diff --git a/.github/ci_scripts/integration_test.sh b/.github/ci_scripts/integration_test.sh index 91f0015a..f35820ff 100755 --- a/.github/ci_scripts/integration_test.sh +++ b/.github/ci_scripts/integration_test.sh @@ -5,7 +5,7 @@ set -o nounset # abort on unbound variable set -o pipefail # don't hide errors within pipes cd example -# TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh +TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh bazel test ... diff --git a/DEVELOPING.md b/DEVELOPING.md new file mode 100644 index 00000000..0824b1ab --- /dev/null +++ b/DEVELOPING.md @@ -0,0 +1,106 @@ +# Building + +Run this first to build everything. + +```bash +./prepare_all_apps.sh +``` + +# Testing + +Run the following to test using the example repo: + +``` +.github/ci_scripts/integration_test.sh +``` + +Or export `REPO_PATH` to be the path of the Bazel repo your working on here... + +```bash +# Pick jvm or python + +ECOSYSTEM=jvm +ECOSYSTEM=python +ECOSYSTEM=protos + +cargo run --bin bzl_gen_build_driver --release -- \ + --input-path build_tools/lang_support/create_lang_build_files/bazel_${ECOSYSTEM}_modules.json \ + --working-directory $REPO_PATH \ + --cache-path ~/.cache/bazel_codegen \ + extract \ + --extractor protos:/tmp/bzl-gen-build/protos-entity-extractor \ + --extractor java:/tmp/bzl-gen-build/java-entity-extractor \ + --extractor scala:/tmp/bzl-gen-build/scala-entity-extractor \ + --extractor python:/tmp/bzl-gen-build/python-entity-extractor \ + --extracted-mappings /tmp/extracted_mappings.${ECOSYSTEM}.json + +cargo run --bin bzl_gen_build_driver --release -- \ + --input-path build_tools/lang_support/create_lang_build_files/bazel_${ECOSYSTEM}_modules.json \ + --working-directory $REPO_PATH \ + --cache-path ~/.cache/bazel_codegen \ + extract-defs \ + --extracted-mappings /tmp/extracted_mappings.${ECOSYSTEM}.json \ + --extracted-defs /tmp/extracted_defs.${ECOSYSTEM}.json + +cargo run --bin bzl_gen_build_driver --release -- \ + --input-path build_tools/lang_support/create_lang_build_files/bazel_${ECOSYSTEM}_modules.json \ + --working-directory $REPO_PATH \ + --cache-path ~/.cache/bazel_codegen \ + build-graph \ + --extracted-mappings /tmp/extracted_mappings.${ECOSYSTEM}.json \ + --extracted-defs /tmp/extracted_defs.${ECOSYSTEM}.json \ + --graph-out /tmp/graph_data.${ECOSYSTEM}.json + +cargo run --bin bzl_gen_build_driver --release -- \ + --input-path build_tools/lang_support/create_lang_build_files/bazel_${ECOSYSTEM}_modules.json \ + --working-directory $REPO_PATH \ + --cache-path ~/.cache/bazel_codegen \ + print-build \ + --graph-data /tmp/graph_data.${ECOSYSTEM}.json +``` + +Shared API type from languages: + +```javascript + { + # A remote label @pip_... or local repo relative path src/a/b/c + "label_or_repo_path": "" + + # List of objects defined in this file/target + "defs": [], + + # List of references to other things defined in this target + "refs": [], + + # List of commands that can be supplied to bzl_build_gen + # These are for special cases + # + # `ref` Add a manual reference to the specified def even though it didn't exist in this file + # `unref` Remove a reference seen in this file + # `def` Add a manual definition in this file on something that isn't visible to the parsers of the file + # `undef` Remove a definition as seen in this file + # `runtime_ref` Add a runtime dependency in this file. This is a colored edge that can be upgraded to a ref if another file or something in the file uses it. + # + # value(SrcDirective::Ref, tag("ref")), + # value(SrcDirective::Unref, tag("unref")), + # value(SrcDirective::Def, tag("def")), + # value(SrcDirective::Undef, tag("undef")), + # value(SrcDirective::RuntimeRef, tag("runtime_ref")), + # value(SrcDirective::RuntimeUnref, tag("runtime_unref")), + bzl_gen_build_commands: [] + } +``` + + +# Implications on the rules/macros used in the repo.. + +We expect to be able to supply the following keys generically: +- `srcs` , list of per language source files +- `deps` , list of dependencies from the graph +- `runtime_deps`, list of dependencies used at runtime, but not compile time. (For some languages like python, a macro wrapping py_library can help merge this into deps) + +# Testing on the repo side + +```bash +TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build ./bazel run build_tools/lang_support/create_lang_build_files:regenerate_jvm_build_files +``` diff --git a/crates/Cargo.lock b/crates/Cargo.lock index 3136ae36..863932bd 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -267,7 +267,7 @@ dependencies = [ "ignore", "lazy_static", "log", - "pretty_env_logger", + "pretty_env_logger 0.5.0", "rustpython-ast", "rustpython-parser", "serde", @@ -311,6 +311,23 @@ dependencies = [ "zip", ] +[[package]] +name = "bzl_gen_protobuf_extractor" +version = "0.1.0" +dependencies = [ + "anyhow", + "bzl_gen_build_shared_types", + "clap", + "log", + "pretty_env_logger 0.4.0", + "serde", + "serde_json", + "tempfile", + "tokio", + "tree-sitter", + "tree-sitter-proto", +] + [[package]] name = "bzl_gen_python_extractor" version = "0.1.0" @@ -323,7 +340,7 @@ dependencies = [ "ignore", "lazy_static", "log", - "pretty_env_logger", + "pretty_env_logger 0.5.0", "rustpython-ast", "rustpython-parser", "serde", @@ -509,13 +526,26 @@ dependencies = [ "log", ] +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + [[package]] name = "env_logger" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" dependencies = [ - "humantime", + "humantime 2.1.0", "is-terminal", "log", "regex", @@ -762,6 +792,15 @@ dependencies = [ "digest", ] +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + [[package]] name = "humantime" version = "2.1.0" @@ -1222,13 +1261,23 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "pretty_env_logger" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" +dependencies = [ + "env_logger 0.7.1", + "log", +] + [[package]] name = "pretty_env_logger" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c" dependencies = [ - "env_logger", + "env_logger 0.10.0", "log", ] @@ -1241,6 +1290,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.29" @@ -1743,6 +1798,25 @@ dependencies = [ "syn", ] +[[package]] +name = "tree-sitter" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad726ec26496bf4c083fff0f43d4eb3a2ad1bba305323af5ff91383c0b6ecac0" +dependencies = [ + "cc", + "regex", +] + +[[package]] +name = "tree-sitter-proto" +version = "0.0.1" +source = "git+https://github.com/mitchellh/tree-sitter-proto.git?rev=42d82fa18f8afe59b5fc0b16c207ee4f84cb185f#42d82fa18f8afe59b5fc0b16c207ee4f84cb185f" +dependencies = [ + "cc", + "tree-sitter", +] + [[package]] name = "twox-hash" version = "1.6.3" diff --git a/crates/Cargo.toml b/crates/Cargo.toml index 67ad51a4..b7354a97 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -2,6 +2,7 @@ members = [ "driver", + "protobuf_extractor", "python_extractor", "python_utilities", "shared_types", diff --git a/crates/README.md b/crates/README.md deleted file mode 100644 index f5686ce2..00000000 --- a/crates/README.md +++ /dev/null @@ -1,69 +0,0 @@ - -Export REPO_PATH to be the path of the bazel repo your working on here... - -``` - -# Pick jvm or python - -ECOSYSTEM=jvm -ECOSYSTEM=python - -``` -cargo run --bin bzl_gen_build_driver --release -- --active-ecosystem $ECOSYSTEM --input-path build_tools/lang_support/create_lang_build_files/bazel_${ECOSYSTEM}_modules.json --working-directory $REPO_PATH --cache-path ~/.cache/bazel_codegen extract --scala-extractor /tmp/bzl-gen-build/scala-entity-extractor --java-extractor /tmp/bzl-gen-build/java-entity-extractor --python-extractor /tmp/bzl-gen-build/python-entity-extractor --extracted-mappings /tmp/extracted_mappings.${ECOSYSTEM}.json -```= - - -``` -cargo run --bin bzl_gen_build_driver --release -- --active-ecosystem $ECOSYSTEM --input-path build_tools/lang_support/create_lang_build_files/bazel_${ECOSYSTEM}_modules.json --working-directory $REPO_PATH --cache-path ~/.cache/bazel_codegen extract-defs --extracted-mappings /tmp/extracted_mappings.${ECOSYSTEM}.json --extracted-defs /tmp/extracted_defs.${ECOSYSTEM}.json - -``` - - -``` -cargo run --bin bzl_gen_build_driver --release -- --active-ecosystem $ECOSYSTEM --input-path build_tools/lang_support/create_lang_build_files/bazel_${ECOSYSTEM}_modules.json --working-directory $REPO_PATH --cache-path ~/.cache/bazel_codegen build-graph --extracted-mappings /tmp/extracted_mappings.${ECOSYSTEM}.json --extracted-defs /tmp/extracted_defs.${ECOSYSTEM}.json --graph-out /tmp/graph_data.${ECOSYSTEM}.json -``` - -``` -cargo run --bin bzl_gen_build_driver --release -- --active-ecosystem $ECOSYSTEM --input-path build_tools/lang_support/create_lang_build_files/bazel_${ECOSYSTEM}_modules.json --working-directory $REPO_PATH --cache-path ~/.cache/bazel_codegen print-build --graph-data /tmp/graph_data.${ECOSYSTEM}.json -``` - - -Shared API type from languages: - -``` - { - # A remote label @pip_... or local repo relative path src/a/b/c - "label_or_repo_path": "" - - # List of objects defined in this file/target - "defs": [], - - # List of references to other things defined in this target - "refs": [], - - # List of commands that can be supplied to bzl_build_gen - # These are for special cases - # - # `ref` Add a manual reference to the specified def even though it didn't exist in this file - # `unref` Remove a reference seen in this file - # `def` Add a manual definition in this file on something that isn't visible to the parsers of the file - # `undef` Remove a definition as seen in this file - # `runtime_ref` Add a runtime dependency in this file. This is a colored edge that can be upgraded to a ref if another file or something in the file uses it. - # - # value(SrcDirective::Ref, tag("ref")), - # value(SrcDirective::Unref, tag("unref")), - # value(SrcDirective::Def, tag("def")), - # value(SrcDirective::Undef, tag("undef")), - # value(SrcDirective::RuntimeRef, tag("runtime_ref")), - # value(SrcDirective::RuntimeUnref, tag("runtime_unref")), - bzl_gen_build_commands: [] - } -``` - - -# Implications on the rules/macros used in the repo.. - -We expect to be able to supply the following keys generically: -- `srcs` , list of per language source files -- `deps` , list of dependencies from the graph -- `runtime_deps`, list of dependencies used at runtime, but not compile time. (For some languages like python, a macro wrapping py_library can help merge this into deps) diff --git a/crates/driver/src/print_build.rs b/crates/driver/src/print_build.rs index 9661e2e4..44b7a0a6 100644 --- a/crates/driver/src/print_build.rs +++ b/crates/driver/src/print_build.rs @@ -474,6 +474,12 @@ where } apply_attr_string_lists(&mut extra_kv_pairs, &graph_node.node_metadata); + // before we give extra_kv_pairs away to make the main target, + // we need to clone deps here for a later use in secondaries. + let deps = extra_kv_pairs + .entry("deps".to_string()) + .or_default() + .clone(); if use_rglob { let target = TargetEntry { name: target_name.clone(), @@ -571,6 +577,88 @@ where t.entries.push(target); } + + fn apply_secondary_rules( + target_entries: &mut TargetEntries, + module_config: &ModuleConfig, + parent_target_name: &str, + parent_include_src: &Vec, + parent_deps: &Vec, + ) { + for (k, build_config) in module_config.build_config.secondary_rules.iter() { + let sec_target_name = format!("{}_{}", parent_target_name, k); + let mut required_load = HashMap::default(); + let mut srcs = Option::default(); + for h in build_config.headers.iter() { + required_load.insert( + Arc::new(h.load_from.clone()), + vec![Arc::new(h.load_value.clone())], + ); + } + let mut extra_kv_pairs: HashMap> = HashMap::default(); + for (k, lst) in build_config.extra_key_to_list.iter() { + let vs = lst.iter().flat_map(|v| { + eval_extra_var(v, parent_target_name, parent_include_src, parent_deps) + }).collect::>(); + match k.as_str() { + "srcs" => srcs = Some(SrcType::List(vs)), + _ => append_key_values(&mut extra_kv_pairs, &k, &vs) + } + } + target_entries.entries.push(TargetEntry { + name: sec_target_name.clone(), + extra_kv_pairs: extra_kv_pairs + .into_iter() + .map(|(k, mut v)| { + v.sort(); + v.dedup(); + (k, v) + }) + .collect(), + extra_k_strs: build_config.extra_key_to_value.clone() + .into_iter() + .map(|(k, v)| { + (k, v) + }) + .collect(), + required_load: required_load.clone(), + visibility: None, + srcs: srcs, + target_type: Arc::new(build_config.function_name.clone()), + }); + } + } + + // This expands `${name}` etc appearing inside of the extra_key_to_list value + // with the name of the parent target. + fn eval_extra_var( + value: &String, + parent_target_name: &str, + parent_include_src: &Vec, + parent_deps: &Vec, + ) -> Vec { + if value.contains("${name}") { + vec![value.replace("${name}", parent_target_name)] + } else if value.contains("${srcs}") { + parent_include_src.clone() + .into_iter() + .map(|v| { + value.replace("${srcs}", &v) + }) + .collect() + } else if value.contains("${deps}") { + parent_deps.clone() + .into_iter() + .map(|v| { + value.replace("${deps}", &v) + }) + .collect() + } else { + vec![value.to_string()] + } + } + + apply_secondary_rules(&mut t, module_config, &target_name, &parent_include_src, &deps); Ok(t) } @@ -700,6 +788,7 @@ pub async fn print_build( #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeMap; use crate::Commands::PrintBuild; use crate::build_config::{BuildConfig, BuildLoad, GrpBuildConfig}; use crate::build_graph::NodeType; @@ -735,7 +824,62 @@ mod tests { extra_key_to_value: HashMap::default() }), test: None, - binary_application: None + binary_application: None, + secondary_rules: BTreeMap::default(), + }, + main_roots: vec!["src/main/protos".to_string()], + test_roots: vec!["src/test/protos".to_string()] + } + )]), + includes: vec![], + path_directives: vec![] + } + } + + fn example_project_conf_with_secondaries() -> ProjectConf { + ProjectConf { + configurations: HashMap::from([( + "protos".to_string(), + ModuleConfig { + file_extensions: vec!["proto".to_string()], + build_config: BuildConfig { + main: Some(GrpBuildConfig { + headers: vec![ + BuildLoad { + load_from: "@rules_proto//proto:defs.bzl".to_string(), + load_value: "proto_library".to_string(), + } + ], + function_name: "proto_library".to_string(), + extra_key_to_list: HashMap::default(), + extra_key_to_value: HashMap::default() + }), + test: None, + binary_application: None, + secondary_rules: BTreeMap::from([ + ("java".to_string(), GrpBuildConfig { + headers: vec![], + function_name: "java_proto_library".to_string(), + extra_key_to_list: HashMap::from([ + ("deps".to_string(), vec![":${name}".to_string()]), + ]), + extra_key_to_value: HashMap::default() + }), + ("py".to_string(), GrpBuildConfig { + headers: vec![ + BuildLoad { + load_from: "@com_google_protobuf//:protobuf.bzl".to_string(), + load_value: "py_proto_library".to_string(), + } + ], + function_name: "py_proto_library".to_string(), + extra_key_to_list: HashMap::from([ + ("srcs".to_string(), vec!["${srcs}".to_string()]), + ("deps".to_string(), vec!["${deps}_py".to_string()]), + ]), + extra_key_to_value: HashMap::default() + }), + ]), }, main_roots: vec!["src/main/protos".to_string()], test_roots: vec!["src/test/protos".to_string()] @@ -766,6 +910,48 @@ filegroup( proto_library( name='protos', srcs=[':protos_files'], + deps=[], + visibility=['//visibility:public'] +) + "#, + ).await + } + + #[tokio::test] + async fn test_generate_targets_with_secondaries() -> Result<(), Box> { + let mut build_graph =GraphNode::default(); + build_graph.node_type = NodeType::RealNode; + test_generate_targets_base( + example_project_conf_with_secondaries(), + build_graph, + "src/main/protos".to_string(), + 4, + r#"load('@com_google_protobuf//:protobuf.bzl', 'py_proto_library') +load('@rules_proto//proto:defs.bzl', 'proto_library') + +filegroup( + name='protos_files', + srcs=glob(include=['**/*.proto']), + visibility=['//visibility:public'] +) + +proto_library( + name='protos', + srcs=[':protos_files'], + deps=[], + visibility=['//visibility:public'] +) + +java_proto_library( + name='protos_java', + deps=[':protos'], + visibility=['//visibility:public'] +) + +py_proto_library( + name='protos_py', + srcs=[':protos_files'], + deps=[], visibility=['//visibility:public'] ) "#, diff --git a/crates/protobuf_extractor/Cargo.toml b/crates/protobuf_extractor/Cargo.toml new file mode 100644 index 00000000..c17f115e --- /dev/null +++ b/crates/protobuf_extractor/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "bzl_gen_protobuf_extractor" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.66" +tokio = { version = "1.22.0", features = ["full"] } +clap = { version = "4.0.26", features = ["derive"] } +serde = { version = "1.0.147", features = ["derive"] } +serde_json = "1.0.88" +pretty_env_logger = "0.4.0" +log = "0.4.17" +tree-sitter = "0.19.3" +tree-sitter-proto = { git = "https://github.com/mitchellh/tree-sitter-proto.git", rev = "42d82fa18f8afe59b5fc0b16c207ee4f84cb185f"} + +[dev-dependencies] +tempfile = "3.3.0" + +[dependencies.bzl_gen_build_shared_types] +path = "../shared_types" + diff --git a/crates/protobuf_extractor/src/extract_protobuf_imports.rs b/crates/protobuf_extractor/src/extract_protobuf_imports.rs new file mode 100644 index 00000000..b7ea0383 --- /dev/null +++ b/crates/protobuf_extractor/src/extract_protobuf_imports.rs @@ -0,0 +1,62 @@ +use anyhow::{bail, Result}; +use tree_sitter::Parser; + +#[derive(Debug, PartialEq)] +pub struct ProtobufSource { + pub imports: Vec +} + +impl ProtobufSource { + pub fn parse(source: &str, _source_path: &str) -> Result { + let mut buf = Vec::default(); + let mut parser = Parser::new(); + parser.set_language(tree_sitter_proto::language())?; + let tree = match parser.parse(source, None) { + Some(tree) => tree, + None => bail!("parse failed"), + }; + let root_node = tree.root_node(); + let mut cursor = root_node.walk(); + let bytes = source.as_bytes(); + for child_node in root_node.children(&mut cursor) { + match child_node.kind() { + "import" => + { + for path in child_node.children_by_field_name("path", &mut child_node.walk()) { + let mut quoted = path.utf8_text(bytes)?.chars(); + // Remove the quotation marks + quoted.next(); + quoted.next_back(); + buf.push(quoted.as_str().to_string()); + } + } + _ => () + }; + } + Ok(ProtobufSource { imports: buf }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_simple_target_entry() -> Result<()> { + let protobuf_source = r#"syntax = "proto3"; + +package page.common; // Requried to generate valid code. + +// Always import protos with a full path relative to the WORKSPACE file. +import "page/common/src/proto/zip_code.proto"; + +message Address { + // string city = 1; + ZipCode zip_code = 2; +}"#; + let parsed = ProtobufSource::parse(protobuf_source, "tmp.proto")?; + let expected = vec!["page/common/src/proto/zip_code.proto".to_string()]; + assert_eq!(parsed.imports, expected); + Ok(()) + } +} diff --git a/crates/protobuf_extractor/src/main.rs b/crates/protobuf_extractor/src/main.rs new file mode 100644 index 00000000..fa23ee3b --- /dev/null +++ b/crates/protobuf_extractor/src/main.rs @@ -0,0 +1,114 @@ +use std::{collections::HashSet, path::PathBuf, time::Instant}; +use anyhow::{Context, Result}; +use bzl_gen_build_shared_types::api::extracted_data::{DataBlock, ExtractedData}; +use clap::Parser; +use log::debug; + +mod extract_protobuf_imports; +use extract_protobuf_imports::ProtobufSource; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +pub struct Opt { + #[clap(long)] + /// comma sepearted list of input files + relative_input_paths: String, + + #[clap(long)] + /// comma sepearted list of input files + working_directory: PathBuf, + + #[clap(long)] + output: PathBuf, + + #[clap(long)] + label_or_repo_path: String, + + #[clap(long)] + disable_ref_generation: bool, + + /// When specified we calculate refs relative to here rather than using a heuristic + #[clap(long)] + import_path_relative_from: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let opt = Opt::parse(); + let mut builder = pretty_env_logger::formatted_timed_builder(); + builder.format_timestamp_nanos(); + builder.target(pretty_env_logger::env_logger::Target::Stderr); + if let Ok(s) = ::std::env::var("RUST_LOG") { + builder.parse_filters(&s); + } else { + builder.parse_filters("warn,protobuf_extractor=info,bzl_gen_build_shared_types=info"); + } + builder.init(); + + let start_time = Instant::now(); + + let mut relative_input_paths: Vec = + if let Some(suffix) = opt.relative_input_paths.strip_prefix('@') { + std::fs::read_to_string(PathBuf::from(suffix))? + .lines() + .map(|e| e.to_string()) + .collect() + } else { + vec![opt.relative_input_paths.clone()] + }; + + relative_input_paths.sort(); + + let mut data_blocks: Vec = Default::default(); + + for relative_path in relative_input_paths { + let input_file = opt.working_directory.join(&relative_path); + let mut refs: HashSet = Default::default(); + let mut defs: HashSet = Default::default(); + let bzl_gen_build_commands: HashSet = Default::default(); + + let input_str = std::fs::read_to_string(&input_file).with_context(|| { + format!( + "While attempting to load up file: {:? + }", + input_file + ) + })?; + + if !opt.disable_ref_generation { + let program = ProtobufSource::parse(&input_str, &relative_path).with_context(|| { + format!( + "Error while parsing file {:? + }", + input_file + ) + })?; + + refs.extend(program.imports); + } + + // See https://protobuf.dev/programming-guides/proto3/#importing + // Protobuf uses file path relative to the workspace as a way of importing: + // import "myproject/other_protos.proto"; + // So here we export the relative path as a "definition". + defs.extend(Some(relative_path.clone())); + + data_blocks.push(DataBlock { + entity_path: relative_path, + defs, + refs, + bzl_gen_build_commands, + }) + } + + let def_refs = ExtractedData { + label_or_repo_path: opt.label_or_repo_path.clone(), + data_blocks, + }; + + tokio::fs::write(opt.output, serde_json::to_string_pretty(&def_refs)?).await?; + + debug!("took {:?}", start_time.elapsed()); + + Ok(()) +} diff --git a/crates/shared_types/src/build_config.rs b/crates/shared_types/src/build_config.rs index 5f4557c2..248487fa 100644 --- a/crates/shared_types/src/build_config.rs +++ b/crates/shared_types/src/build_config.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use serde::{Deserialize, Serialize}; @@ -12,6 +12,9 @@ pub struct BuildConfig { #[serde(default)] pub binary_application: Option, + + #[serde(default)] + pub secondary_rules: BTreeMap, } impl BuildConfig { diff --git a/crates/shared_types/src/project_conf.rs b/crates/shared_types/src/project_conf.rs index 0a07bc8b..3e84fb5b 100644 --- a/crates/shared_types/src/project_conf.rs +++ b/crates/shared_types/src/project_conf.rs @@ -102,7 +102,7 @@ pub struct PathMatcher {} #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::collections::{BTreeMap, HashMap}; use crate::{ build_config::{BuildConfig, GrpBuildConfig}, @@ -166,7 +166,8 @@ mod tests { extra_key_to_value: HashMap::default() }), test: None, - binary_application: None + binary_application: None, + secondary_rules: BTreeMap::default() }, main_roots: vec!["src/main/python".to_string()], test_roots: vec!["src/test/python".to_string()] diff --git a/example/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh b/example/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh index 670cbe69..01c32bab 100755 --- a/example/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh +++ b/example/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh @@ -149,6 +149,7 @@ function run_system_apps() { --input-path $CFG \ --working-directory $REPO_ROOT \ --cache-path ${BZL_GEN_BUILD_CACHE_PATH} extract \ + --extractor protos:${TOOLING_WORKING_DIRECTORY}/protos-entity-extractor \ --extractor scala:${TOOLING_WORKING_DIRECTORY}/scala-entity-extractor \ --external-generated-root ${TMP_WORKING_STATE}/external_files \ --extractor java:${TOOLING_WORKING_DIRECTORY}/java-entity-extractor \ diff --git a/prepare_all_apps.sh b/prepare_all_apps.sh index 547dbbde..7d5a6b90 100755 --- a/prepare_all_apps.sh +++ b/prepare_all_apps.sh @@ -38,6 +38,9 @@ cp ${RUST_TARGET_DIR}/bzl_gen_build_driver $PREPARE_ALL_OUTPUT_DIR/system-driver rm -f $PREPARE_ALL_OUTPUT_DIR/python-entity-extractor || true cp ${RUST_TARGET_DIR}/bzl_gen_python_extractor $PREPARE_ALL_OUTPUT_DIR/python-entity-extractor +rm -f $PREPARE_ALL_OUTPUT_DIR/protobuf-extractor || true +cp ${RUST_TARGET_DIR}/bzl_gen_protobuf_extractor $PREPARE_ALL_OUTPUT_DIR/protos-entity-extractor + echo "workspace(name = \"external_build_tooling_gen\")" > $PREPARE_ALL_OUTPUT_DIR/WORKSPACE cat > $PREPARE_ALL_OUTPUT_DIR/BUILD.bazel <