From 455fa41191dc37ac0a34e827b32a73b4ee80e6a2 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 24 Jul 2023 16:16:46 -0400 Subject: [PATCH 1/3] Example repo Problem ------- In the current state bzl-gen-build is not really usable outside of our own usage since it's relying on custom scripts and macros. This is often necessitated by our tendency to use old version of Python etc. Solution -------- This adds an example repository to reimplement some of the scripts and configurations using open source rules_python etc. --- .github/ci_scripts/integration_test.sh | 10 + .github/workflows/test.yml | 31 ++- .gitignore | 2 + crates/python_extractor/src/main.rs | 20 +- example/.bazelrc | 1 + example/.bazelversion | 1 + example/BUILD.bazel | 11 ++ example/MODULE.bazel | 20 ++ example/WORKSPACE | 32 +++ example/build_tools/BUILD.bazel | 0 .../bazel_rules/wheel_scanner/BUILD.bazel | 5 + .../wheel_scanner/py_build_commands.py | 87 +++++++++ .../wheel_scanner/wheel_scanner.bzl | 67 +++++++ .../bazel_protos_modules.json | 48 +++++ .../bazel_python_modules.json | 26 +++ .../bzl_gen_build_common.sh | 184 ++++++++++++++++++ .../create_lang_build_files/gen/WORKSPACE | 0 .../regenerate_protos_build_files.sh | 18 ++ .../regenerate_python_build_files.sh | 35 ++++ example/requirements.in | 4 + example/requirements_lock_3_9.txt | 93 +++++++++ example/src/main/protos/com/example/A.proto | 9 + .../src/main/protos/com/example/BUILD.bazel | 6 + .../src/main/protos/com/example/foo/B.proto | 11 ++ .../main/protos/com/example/foo/BUILD.bazel | 6 + .../main/python/com/example/hello/BUILD.bazel | 2 + .../main/python/com/example/hello/hello.py | 10 + example/src/test/java/com/example/BUILD.bazel | 8 + example/src/test/java/com/example/Test1.java | 17 ++ .../test/python/com/example/hello/BUILD.bazel | 1 + .../test/python/com/example/hello/hello.py | 10 + prepare_all_apps.sh | 4 +- 32 files changed, 770 insertions(+), 9 deletions(-) create mode 100755 .github/ci_scripts/integration_test.sh create mode 100644 example/.bazelrc create mode 100644 example/.bazelversion create mode 100644 example/BUILD.bazel create mode 100644 example/MODULE.bazel create mode 100644 example/WORKSPACE create mode 100644 example/build_tools/BUILD.bazel create mode 100644 example/build_tools/bazel_rules/wheel_scanner/BUILD.bazel create mode 100644 example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py create mode 100644 example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl create mode 100644 example/build_tools/lang_support/create_lang_build_files/bazel_protos_modules.json create mode 100644 example/build_tools/lang_support/create_lang_build_files/bazel_python_modules.json create mode 100755 example/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh create mode 100644 example/build_tools/lang_support/create_lang_build_files/gen/WORKSPACE create mode 100644 example/build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh create mode 100755 example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh create mode 100644 example/requirements.in create mode 100644 example/requirements_lock_3_9.txt create mode 100644 example/src/main/protos/com/example/A.proto create mode 100644 example/src/main/protos/com/example/BUILD.bazel create mode 100644 example/src/main/protos/com/example/foo/B.proto create mode 100644 example/src/main/protos/com/example/foo/BUILD.bazel create mode 100644 example/src/main/python/com/example/hello/BUILD.bazel create mode 100644 example/src/main/python/com/example/hello/hello.py create mode 100644 example/src/test/java/com/example/BUILD.bazel create mode 100644 example/src/test/java/com/example/Test1.java create mode 100644 example/src/test/python/com/example/hello/BUILD.bazel create mode 100644 example/src/test/python/com/example/hello/hello.py diff --git a/.github/ci_scripts/integration_test.sh b/.github/ci_scripts/integration_test.sh new file mode 100755 index 00000000..b96b1900 --- /dev/null +++ b/.github/ci_scripts/integration_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +cd example +# TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh +TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh +bazel test ... diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0af9509e..31b937e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,12 +8,22 @@ jobs: strategy: fail-fast: true matrix: - os: [macos-latest, ubuntu-latest] include: - os: macos-latest platform: macos + jobtype: 1 + - os: macos-latest + platform: macos + jobtype: 3 + - os: ubuntu-latest + platform: linux + jobtype: 1 - os: ubuntu-latest platform: linux + jobtype: 2 + - os: ubuntu-latest + platform: linux + jobtype: 3 steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -33,7 +43,18 @@ jobs: distribution: temurin java-version: 8 cache: sbt - - run: cd crates; cargo test --all-features - - run: cd crates; cargo test - - run: cd language_generators/scala-defref-extractor; ./sbt "test; scalafmtCheckAll" - - run: ./prepare_all_apps.sh + - name: Bazel cache + uses: actions/cache@v3 + with: + path: "/home/runner/.cache/bazel" + key: bazel-${{ hashFiles('**/.bazelrc', '**/.bazeliskrc', '**/WORKSPACE', '**/WORKSPACE.bazel', '**/MODULE.bazel') }} + - if: ${{ matrix.jobtype == 1 }} + run: cd crates; cargo test --all-features + - if: ${{ matrix.jobtype == 1 }} + run: cd crates; cargo test + - if: ${{ matrix.jobtype == 2 }} + run: cd language_generators/scala-defref-extractor; ./sbt "test; scalafmtCheckAll" + - if: ${{ matrix.jobtype == 3 }} + run: | + ./prepare_all_apps.sh + .github/ci_scripts/integration_test.sh diff --git a/.gitignore b/.gitignore index 5c21490b..47bfb005 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ .vscode .metals + +bazel-* diff --git a/crates/python_extractor/src/main.rs b/crates/python_extractor/src/main.rs index 78eab3fc..e82ed090 100644 --- a/crates/python_extractor/src/main.rs +++ b/crates/python_extractor/src/main.rs @@ -123,12 +123,16 @@ async fn main() -> Result<()> { } fn expand_path_to_defs_from_offset(from_given_path: &str, path: &str) -> Vec { - if let Some(rem) = path.strip_prefix(from_given_path) { + if let Some(rem0) = path.strip_prefix(from_given_path) { + let rem = match rem0.strip_prefix("site-packages/") { + Some(x) => x, + None => rem0 + }; if let Some(e) = rem.strip_suffix(".py") { let targ = e.replace('/', "."); if let Some(p) = targ.strip_suffix(".__init__") { - return vec![targ.clone(), p.to_string()]; + return vec![p.to_string(), targ.clone()]; } else { return vec![targ]; } @@ -187,4 +191,16 @@ mod tests { expected ); } + + #[test] + fn expand_site_packages_path_to_defs_test() { + let mut expected = vec!["pytz", "pytz.__init__"]; + expected.sort(); + expected.dedup(); + + assert_eq!( + expand_path_to_defs_from_offset("/tmp/aaa/", "/tmp/aaa/site-packages/pytz/__init__.py"), + expected + ); + } } diff --git a/example/.bazelrc b/example/.bazelrc new file mode 100644 index 00000000..3ce91d27 --- /dev/null +++ b/example/.bazelrc @@ -0,0 +1 @@ +common --enable_bzlmod diff --git a/example/.bazelversion b/example/.bazelversion new file mode 100644 index 00000000..024b066c --- /dev/null +++ b/example/.bazelversion @@ -0,0 +1 @@ +6.2.1 diff --git a/example/BUILD.bazel b/example/BUILD.bazel new file mode 100644 index 00000000..c693261a --- /dev/null +++ b/example/BUILD.bazel @@ -0,0 +1,11 @@ +load("@pip//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement") +load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test") +load("@python_versions//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requirements") +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") + +compile_pip_requirements_3_9( + name = "requirements_3_9", + extra_args = ["--allow-unsafe"], + requirements_in = "requirements.in", + requirements_txt = "requirements_lock_3_9.txt", +) diff --git a/example/MODULE.bazel b/example/MODULE.bazel new file mode 100644 index 00000000..88f67f84 --- /dev/null +++ b/example/MODULE.bazel @@ -0,0 +1,20 @@ +module( + name = "example", + version = "0.0.0", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.4.1") +bazel_dep(name = "rules_python", version = "0.24.0") +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + python_version = "3.9", +) +use_repo(python, "python_3_9", "python_versions") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "pip", + requirements_lock = "//:requirements_lock_3_9.txt", +) +use_repo(pip, "pip", "pip_39") diff --git a/example/WORKSPACE b/example/WORKSPACE new file mode 100644 index 00000000..5bd9261a --- /dev/null +++ b/example/WORKSPACE @@ -0,0 +1,32 @@ +workspace(name = "example") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +skylib_version = "1.0.3" +http_archive( + name = "bazel_skylib", + sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", + type = "tar.gz", + url = "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib-{}.tar.gz".format(skylib_version, skylib_version), +) + +## Start: Protobuf +http_archive( + name = "rules_proto", + url = "https://github.com/bazelbuild/rules_proto/archive/c0b62f2f46c85c16cb3b5e9e921f0d00e3101934.tar.gz", + sha256 = "e0cab008a9cdc2400a1d6572167bf9c5afc72e19ee2b862d18581051efab42c9", + strip_prefix = "rules_proto-c0b62f2f46c85c16cb3b5e9e921f0d00e3101934", +) + +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + +rules_proto_dependencies() + +rules_proto_toolchains() + +## End: Protobuf + +local_repository( + name = "external_build_tooling_gen", + path = "build_tools/lang_support/create_lang_build_files/gen", +) diff --git a/example/build_tools/BUILD.bazel b/example/build_tools/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/example/build_tools/bazel_rules/wheel_scanner/BUILD.bazel b/example/build_tools/bazel_rules/wheel_scanner/BUILD.bazel new file mode 100644 index 00000000..92ac88b7 --- /dev/null +++ b/example/build_tools/bazel_rules/wheel_scanner/BUILD.bazel @@ -0,0 +1,5 @@ +py_binary( + name = "py_build_commands", + srcs = ["py_build_commands.py"], + visibility = ["//visibility:public"], +) diff --git a/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py b/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py new file mode 100644 index 00000000..014408d5 --- /dev/null +++ b/example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py @@ -0,0 +1,87 @@ +import sys +import json +import os +import zipfile + +BATCH_SIZE = 2000 + +PRELUDE = """ +#!/bin/bash + +set -efo pipefail + +set +x +trap 'echo ERROR in ${BASH_SOURCE[0]}, failed to run command, line with error: $LINENO' ERR + +OUTPUT_BASE="$(bazel info output_base)" +""" +TEMPLATE = """ + +echo -n "Running scan of 3rdparty files in batches, working on batch {output_idx}, with {target_count} targets in it" + +START_BATCH=$(date +%s) + + +set +e +bazel build {targets} \ + --aspects build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl%wheel_scanner_aspect \ + --output_groups=+wheel_scanner_out \ + --override_repository=external_build_tooling_gen=${{BZL_GEN_BUILD_TOOLS_PATH}} \ + --show_result=1000000 2> /tmp/cmd_out +RET=$? +if [ "$RET" != "0" ]; then + cat /tmp/cmd_out + exit $RET +fi + +set +o pipefail +inner_idx=0 +for f in `cat $OUTPUT_BASE/command.log | + grep ".*\.json$" | + sed -e 's/^[^ ]*//' | + sed -e 's/^[^A-Za-z0-9/]*//' | + sed 's/^ *//;s/ *$//'`; do + if [ -f "$f" ]; then + TARGET_PATH="${{BZL_BUILD_GEN_EXTERNAL_FILES_PATH}}/{output_idx}_${{inner_idx}}_wheel_scanner.json" + cp $f $TARGET_PATH + inner_idx=$((inner_idx + 1)) + fi +done + +set -o pipefail +END_BATCH=$(date +%s) + +echo "...complete in $(($END_BATCH-$START_BATCH)) seconds" +""" + + +def __transform_target(t): + return "@%s//:pkg" % (t.lstrip("//external:")) + +def write_command(file, output_idx, command_list): + file.write( + TEMPLATE.format( + targets=" ".join([t for t in command_list if t.endswith("pkg")]), + output_idx=output_idx, + target_count=len(command_list), + ) + ) + + +if __name__ == "__main__": + input_file = sys.argv[1] + output_file_path = sys.argv[2] + external_targets = [] + output_idx = 0 + with open(input_file, "r") as file1: + with open(output_file_path, "w") as output_file: + output_file.write(PRELUDE) + for line in file1.readlines(): + external_targets.append(line.strip()) + if len(external_targets) > BATCH_SIZE: + write_command(output_file, output_idx, external_targets) + output_idx += 1 + external_targets = [] + if len(external_targets) > 0: + write_command(output_file, output_idx, external_targets) + output_idx += 1 diff --git a/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl b/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl new file mode 100644 index 00000000..d7cb9f12 --- /dev/null +++ b/example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl @@ -0,0 +1,67 @@ +def _wheel_scanner_impl(target, ctx): + if not target.label.workspace_name.startswith("rules_python"): + return [] + + # Make sure the rule has a srcs attribute. + out_content = ctx.actions.declare_file("%s_wheel_scanner.json" % (target.label.name)) + + files = ctx.rule.files + all_py_files = [] + all_py_relative_paths = [] + workspace_root = target.label.workspace_root + if hasattr(files, "srcs"): + for file in files.srcs: + basename = file.basename + + # We don't expect to have dependencies on tests + # These generated rules python files seem to have some invalid python syntax too + if not basename.startswith("rules_python_wheel_") and not basename.startswith("test_"): + path = file.path + if not path.startswith(workspace_root): + fail("Didn't have workspace prefix") + all_py_relative_paths.append(path[len(workspace_root) + 1:]) + all_py_files.append(file) + + input_files = ctx.actions.declare_file("%s_wheel_scanner_input_files.txt" % (target.label.name)) + ctx.actions.write( + input_files, + "\n".join(all_py_relative_paths), + ) + + args = ctx.actions.args() + args.add("--disable-ref-generation") + args.add("--label-or-repo-path") + args.add(str(target.label)) + + args.add("--import-path-relative-from") + args.add("%s/" % (target.label.workspace_root)) + args.add("--working-directory") + args.add(target.label.workspace_root) + args.add("--relative-input-paths") + args.add("@%s" % input_files.path) + args.add("--output") + args.add(out_content) + + inputs = [input_files] + inputs.extend(all_py_files) + ctx.actions.run( + outputs = [out_content], + inputs = inputs, + executable = ctx.files._py_exe[0], + mnemonic = "WheelScanner", + arguments = [args], + ) + + return [OutputGroupInfo(wheel_scanner_out = depset([out_content]))] + +wheel_scanner_aspect = aspect( + implementation = _wheel_scanner_impl, + attr_aspects = [], + attrs = { + "_py_exe": attr.label( + default = Label("@external_build_tooling_gen//:python-entity-extractor"), + allow_files = True, + cfg = "host", + ), + }, +) diff --git a/example/build_tools/lang_support/create_lang_build_files/bazel_protos_modules.json b/example/build_tools/lang_support/create_lang_build_files/bazel_protos_modules.json new file mode 100644 index 00000000..c2c9ae6b --- /dev/null +++ b/example/build_tools/lang_support/create_lang_build_files/bazel_protos_modules.json @@ -0,0 +1,48 @@ +{ + "configurations": { + "protos": { + "file_extensions": [ + "proto" + ], + "build_config": { + "main": { + "headers": [ + { + "load_from": "@rules_proto//proto:defs.bzl", + "load_value": "proto_library" + } + ], + "function_name": "proto_library" + }, + "secondary_rules": { + "java": { + "headers": [], + "function_name": "java_proto_library", + "extra_key_to_list": { + "deps": [":${name}"] + } + }, + "py": { + "headers": [ + { + "load_from": "@rules_python//python:proto.bzl", + "load_value": "py_proto_library" + } + ], + "function_name": "py_proto_library", + "extra_key_to_list": { + "deps": [":${name}"] + } + } + } + }, + "main_roots": [ + "src/main/protos" + ], + "test_roots": [ + "src/test/protos" + ], + "path_directives": [] + } + } +} diff --git a/example/build_tools/lang_support/create_lang_build_files/bazel_python_modules.json b/example/build_tools/lang_support/create_lang_build_files/bazel_python_modules.json new file mode 100644 index 00000000..0968fbd6 --- /dev/null +++ b/example/build_tools/lang_support/create_lang_build_files/bazel_python_modules.json @@ -0,0 +1,26 @@ +{ + "configurations": { + "python": { + "file_extensions": [ + "py" + ], + "build_config": { + "main": { + "headers": [], + "function_name": "py_library" + }, + "test": { + "headers": [], + "function_name": "py_test" + } + }, + "main_roots": [ + "src/main/python" + ], + "test_roots": [ + "src/test/python" + ], + "path_directives": [] + } + } + } \ No newline at end of file diff --git a/example/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh b/example/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh new file mode 100755 index 00000000..670cbe69 --- /dev/null +++ b/example/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh @@ -0,0 +1,184 @@ +#!/bin/bash + +set -euo pipefail + +set +x +BZL_GEN_BUILD_VERSION=v0.1-81 +BZL_GEN_BUILD_SOURCE_GITHUB_REPO=bazeltools/bzl-gen-build + +function log(){ + DTE="$(date '+%Y %m %d %H:%M:%S')" + GREEN='\033[0;32m' + NC='\033[0m' + + printf "\n\n$GREEN$DTE - $1$NC\n\n" 1>&2 +} + + +if [ "$(uname -s)" == "Linux" ]; then + export BZL_GEN_PLATFORM='linux-ubuntu-20.04' + export BUILDIFIER_PLATFORM_SUFFIX="-linux-amd64" +elif [ "$(uname -s)" == "Darwin" ]; then + ARCH="$(uname -m)" + if [ "$ARCH" == "arm64" ]; then + export BZL_GEN_PLATFORM='macos-arm64' + export BUILDIFIER_PLATFORM_SUFFIX="-darwin-arm64" + else + export BZL_GEN_PLATFORM='macos-x86' + export BUILDIFIER_PLATFORM_SUFFIX="-darwin-amd64" + fi +else + "Your platform $(uname -s) is unsupported, sorry" + exit 1 +fi + +export BZL_GEN_BUILD_CACHE_PATH=/tmp/bzl_gen_build/code + +if [ -z "${BZL_GEN_BUILD_TOOLS_PATH:-}" ]; then + BZL_GEN_BUILD_TOOLS_PATH=$HOME/.cache/bzl_gen_build/tools +fi +mkdir -p "$BZL_GEN_BUILD_TOOLS_PATH" +BZL_BUILD_GEN_TOOLS_URL=https://github.com/${BZL_GEN_BUILD_SOURCE_GITHUB_REPO}/releases/download/${BZL_GEN_BUILD_VERSION}/bzl-gen-build-${BZL_GEN_PLATFORM}.tgz +BZL_BUILD_GEN_TOOLS_SHA_URL=https://github.com/${BZL_GEN_BUILD_SOURCE_GITHUB_REPO}/releases/download/${BZL_GEN_BUILD_VERSION}/bzl-gen-build-${BZL_GEN_PLATFORM}.tgz.sha256 +BZL_BUILD_GEN_TOOLS_LOCAL_PATH="${BZL_GEN_BUILD_TOOLS_PATH}/${BZL_GEN_BUILD_VERSION}" + + +if [ -z "${TOOLING_WORKING_DIRECTORY:-}" ]; then + if [ ! -d "$BZL_BUILD_GEN_TOOLS_LOCAL_PATH" ]; then + log "Fetching bzl-gen-build ${BZL_GEN_BUILD_VERSION}" + rm -rf "$BZL_BUILD_GEN_TOOLS_LOCAL_PATH" &> /dev/null || true + DOWNLOAD_PATH="${BZL_BUILD_GEN_TOOLS_LOCAL_PATH}.tgz" + 1>&2 fetch_binary "${DOWNLOAD_PATH}" "$BZL_BUILD_GEN_TOOLS_URL" "$BZL_BUILD_GEN_TOOLS_SHA_URL" + TMP_P=${BZL_BUILD_GEN_TOOLS_LOCAL_PATH}.tmp + mkdir -p $TMP_P + cd $TMP_P + set +e + tar zxvf $DOWNLOAD_PATH &> /dev/null + RET=$? + if [ "$RET" != "0" ]; then + tar zxvf $DOWNLOAD_PATH 1>&2 + fi + set -e + cd $REPO_ROOT + mv $TMP_P $BZL_BUILD_GEN_TOOLS_LOCAL_PATH + fi +fi + +if [ -z "$GEN_FLAVOR" ]; then + echo "Need to have specified GEN_FLAVOR before sourcing this" +fi + +if [ -n "${INVOKED_VIA_BAZEL:-}" ]; then + REPO_ROOT="$BUILD_WORKING_DIRECTORY" +else + REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )" +fi + + +cd $REPO_ROOT + +log "Initial bazel command, may sync tools." + + +set +e +bazel info output_base &> /dev/null +RET=$? +set -e +if [ "$RET" != "0" ]; then + bazel info output_base +fi + +export OUTPUT_BASE="$(bazel info output_base 2> /dev/null)" + +if [ -z "$OUTPUT_BASE" ] || [ ! -d "$OUTPUT_BASE" ]; then + echo "Invalid output base value $OUTPUT_BASE" 1>&2 + bazel info output_base + exit 1 +fi +if [ ! -f WORKSPACE ]; then + echo "Attempted to get into the bazel root and find the workspace, but we failed, we are in $PWD" 1>&2 + exit 1 +fi + +export TMP_WORKING_STATE=/tmp/build_gen_${GEN_FLAVOR} +export TMP_DOWNLOAD_CACHE=/tmp/build_downloads + +mkdir -p $TMP_DOWNLOAD_CACHE + +mkdir -p $BZL_GEN_BUILD_CACHE_PATH +rm -rf $TMP_WORKING_STATE +mkdir -p $TMP_WORKING_STATE +export BZL_BUILD_GEN_EXTERNAL_FILES_PATH=$TMP_WORKING_STATE/external_files +mkdir -p $BZL_BUILD_GEN_EXTERNAL_FILES_PATH + +if [ -z "${TOOLING_WORKING_DIRECTORY:-}" ]; then + export TOOLING_WORKING_DIRECTORY="$BZL_BUILD_GEN_TOOLS_LOCAL_PATH" +else + export TOOLING_WORKING_DIRECTORY="$TOOLING_WORKING_DIRECTORY" +fi + +if [ ! -d "$TOOLING_WORKING_DIRECTORY" ]; then + echo "$TOOLING_WORKING_DIRECTORY should point at the location of an unpacked bzl_build_gen config" 1>&2 + exit 1 +fi + +function generate_cache_key() { + rm -f /tmp/bzl_gen_remote_cache_key_builder &> /dev/null || true + set -e + for arg in $@; do + cat $arg >> /tmp/bzl_gen_remote_cache_key_builder + done + echo "$BZL_GEN_BUILD_VERSION" >> /tmp/bzl_gen_remote_cache_key_builder + shasum -a 256 /tmp/bzl_gen_remote_cache_key_builder | awk '{print $1}' +} + +function run_system_apps() { + CFG="$1" + if [ -z "$CFG" ]; then + echo "Must supply a config file, like the bazel_jvm_modules.json" 1>&2 + exit 1 + fi + + if [ ! -f "$CFG" ]; then + echo "Config argument must be a file, given $CFG" 1>&2 + exit 1 + fi + + set -ex + ${TOOLING_WORKING_DIRECTORY}/system-driver-app \ + --input-path $CFG \ + --working-directory $REPO_ROOT \ + --cache-path ${BZL_GEN_BUILD_CACHE_PATH} extract \ + --extractor scala:${TOOLING_WORKING_DIRECTORY}/scala-entity-extractor \ + --external-generated-root ${TMP_WORKING_STATE}/external_files \ + --extractor java:${TOOLING_WORKING_DIRECTORY}/java-entity-extractor \ + --extractor python:${TOOLING_WORKING_DIRECTORY}/python-entity-extractor \ + --extracted-mappings ${TMP_WORKING_STATE}/extracted_mappings.json + + ${TOOLING_WORKING_DIRECTORY}/system-driver-app \ + --input-path $CFG \ + --working-directory $REPO_ROOT \ + --cache-path ${BZL_GEN_BUILD_CACHE_PATH} \ + extract-defs \ + --extracted-mappings ${TMP_WORKING_STATE}/extracted_mappings.json \ + --extracted-defs ${TMP_WORKING_STATE}/extracted_defs.json + + ${TOOLING_WORKING_DIRECTORY}/system-driver-app \ + --input-path $CFG \ + --working-directory $REPO_ROOT \ + --cache-path ${BZL_GEN_BUILD_CACHE_PATH} \ + build-graph \ + --extracted-mappings ${TMP_WORKING_STATE}/extracted_mappings.json \ + --extracted-defs ${TMP_WORKING_STATE}/extracted_defs.json \ + --graph-out ${TMP_WORKING_STATE}/graph_data.json + + ${TOOLING_WORKING_DIRECTORY}/system-driver-app \ + --input-path $CFG \ + --working-directory $REPO_ROOT \ + --cache-path ${BZL_GEN_BUILD_CACHE_PATH} \ + print-build \ + --graph-data ${TMP_WORKING_STATE}/graph_data.json + set +x +} + +cd $REPO_ROOT diff --git a/example/build_tools/lang_support/create_lang_build_files/gen/WORKSPACE b/example/build_tools/lang_support/create_lang_build_files/gen/WORKSPACE new file mode 100644 index 00000000..e69de29b diff --git a/example/build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh b/example/build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh new file mode 100644 index 00000000..88a527bb --- /dev/null +++ b/example/build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +if [ -n "${INVOKED_VIA_BAZEL:-}" ]; then + REPO_ROOT="$BUILD_WORKING_DIRECTORY" +else + REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )" +fi + +GEN_FLAVOR=protos +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh" + +log "generate core build files ($GEN_FLAVOR)" + +run_system_apps "build_tools/lang_support/create_lang_build_files/bazel_${GEN_FLAVOR}_modules.json" diff --git a/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh b/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh new file mode 100755 index 00000000..96459903 --- /dev/null +++ b/example/build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +if [ -n "${INVOKED_VIA_BAZEL:-}" ]; then + REPO_ROOT="$BUILD_WORKING_DIRECTORY" +else + REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )" +fi + +GEN_FLAVOR=python +source "$REPO_ROOT/build_tools/lang_support/create_lang_build_files/bzl_gen_build_common.sh" +set -x + +bazel query '@pip//...' | grep '@pip' > $TMP_WORKING_STATE/external_targets + +# try_fetch_from_remote_cache "remote_python_${CACHE_KEY}" + +# if [ ! -d $TMP_WORKING_STATE/external_files ]; then + # log "cache wasn't ready or populated" + bazel run build_tools/bazel_rules/wheel_scanner:py_build_commands -- $TMP_WORKING_STATE/external_targets $TMP_WORKING_STATE/external_targets_commands.sh + chmod +x ${TMP_WORKING_STATE}/external_targets_commands.sh + mkdir -p $TMP_WORKING_STATE/external_files + if [[ -d $TOOLING_WORKING_DIRECTORY ]]; then + BZL_GEN_BUILD_TOOLS_PATH=$TOOLING_WORKING_DIRECTORY ${TMP_WORKING_STATE}/external_targets_commands.sh + else + BZL_GEN_BUILD_TOOLS_PATH=$BZL_BUILD_GEN_TOOLS_LOCAL_PATH ${TMP_WORKING_STATE}/external_targets_commands.sh + fi + + # update_remote_cache "remote_python_${CACHE_KEY}" +# fi + +run_system_apps "build_tools/lang_support/create_lang_build_files/bazel_${GEN_FLAVOR}_modules.json" diff --git a/example/requirements.in b/example/requirements.in new file mode 100644 index 00000000..8ec115fd --- /dev/null +++ b/example/requirements.in @@ -0,0 +1,4 @@ +--extra-index-url https://pypi.python.org/simple/ + +protobuf +pandas diff --git a/example/requirements_lock_3_9.txt b/example/requirements_lock_3_9.txt new file mode 100644 index 00000000..dedd23c7 --- /dev/null +++ b/example/requirements_lock_3_9.txt @@ -0,0 +1,93 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# bazel run //:requirements_3_9.update +# +--extra-index-url https://pypi.python.org/simple/ + +numpy==1.25.1 \ + --hash=sha256:012097b5b0d00a11070e8f2e261128c44157a8689f7dedcf35576e525893f4fe \ + --hash=sha256:0d3fe3dd0506a28493d82dc3cf254be8cd0d26f4008a417385cbf1ae95b54004 \ + --hash=sha256:0def91f8af6ec4bb94c370e38c575855bf1d0be8a8fbfba42ef9c073faf2cf19 \ + --hash=sha256:1a180429394f81c7933634ae49b37b472d343cccb5bb0c4a575ac8bbc433722f \ + --hash=sha256:1d5d3c68e443c90b38fdf8ef40e60e2538a27548b39b12b73132456847f4b631 \ + --hash=sha256:20e1266411120a4f16fad8efa8e0454d21d00b8c7cee5b5ccad7565d95eb42dd \ + --hash=sha256:247d3ffdd7775bdf191f848be8d49100495114c82c2bd134e8d5d075fb386a1c \ + --hash=sha256:35a9527c977b924042170a0887de727cd84ff179e478481404c5dc66b4170009 \ + --hash=sha256:38eb6548bb91c421261b4805dc44def9ca1a6eef6444ce35ad1669c0f1a3fc5d \ + --hash=sha256:3d7abcdd85aea3e6cdddb59af2350c7ab1ed764397f8eec97a038ad244d2d105 \ + --hash=sha256:41a56b70e8139884eccb2f733c2f7378af06c82304959e174f8e7370af112e09 \ + --hash=sha256:4a90725800caeaa160732d6b31f3f843ebd45d6b5f3eec9e8cc287e30f2805bf \ + --hash=sha256:6b82655dd8efeea69dbf85d00fca40013d7f503212bc5259056244961268b66e \ + --hash=sha256:6c6c9261d21e617c6dc5eacba35cb68ec36bb72adcff0dee63f8fbc899362588 \ + --hash=sha256:77d339465dff3eb33c701430bcb9c325b60354698340229e1dff97745e6b3efa \ + --hash=sha256:791f409064d0a69dd20579345d852c59822c6aa087f23b07b1b4e28ff5880fcb \ + --hash=sha256:9a3a9f3a61480cc086117b426a8bd86869c213fc4072e606f01c4e4b66eb92bf \ + --hash=sha256:c1516db588987450b85595586605742879e50dcce923e8973f79529651545b57 \ + --hash=sha256:c40571fe966393b212689aa17e32ed905924120737194b5d5c1b20b9ed0fb171 \ + --hash=sha256:d412c1697c3853c6fc3cb9751b4915859c7afe6a277c2bf00acf287d56c4e625 \ + --hash=sha256:d5154b1a25ec796b1aee12ac1b22f414f94752c5f94832f14d8d6c9ac40bcca6 \ + --hash=sha256:d736b75c3f2cb96843a5c7f8d8ccc414768d34b0a75f466c05f3a739b406f10b \ + --hash=sha256:e8f6049c4878cb16960fbbfb22105e49d13d752d4d8371b55110941fb3b17800 \ + --hash=sha256:f76aebc3358ade9eacf9bc2bb8ae589863a4f911611694103af05346637df1b7 \ + --hash=sha256:fd67b306320dcadea700a8f79b9e671e607f8696e98ec255915c0c6d6b818503 + # via pandas +pandas==2.0.3 \ + --hash=sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682 \ + --hash=sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc \ + --hash=sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b \ + --hash=sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089 \ + --hash=sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5 \ + --hash=sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26 \ + --hash=sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210 \ + --hash=sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b \ + --hash=sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641 \ + --hash=sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd \ + --hash=sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78 \ + --hash=sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b \ + --hash=sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e \ + --hash=sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061 \ + --hash=sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0 \ + --hash=sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e \ + --hash=sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8 \ + --hash=sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d \ + --hash=sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0 \ + --hash=sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c \ + --hash=sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183 \ + --hash=sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df \ + --hash=sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8 \ + --hash=sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f \ + --hash=sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02 + # via -r requirements.in +protobuf==4.23.4 \ + --hash=sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474 \ + --hash=sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2 \ + --hash=sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b \ + --hash=sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720 \ + --hash=sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12 \ + --hash=sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd \ + --hash=sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0 \ + --hash=sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e \ + --hash=sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9 \ + --hash=sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70 \ + --hash=sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff \ + --hash=sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597 \ + --hash=sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a + # via -r requirements.in +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via pandas +pytz==2023.3 \ + --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ + --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb + # via pandas +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via python-dateutil +tzdata==2023.3 \ + --hash=sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a \ + --hash=sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda + # via pandas diff --git a/example/src/main/protos/com/example/A.proto b/example/src/main/protos/com/example/A.proto new file mode 100644 index 00000000..a44ec3a8 --- /dev/null +++ b/example/src/main/protos/com/example/A.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package com.example; + +option java_multiple_files = true; + +message A { + repeated string tags = 1; +} diff --git a/example/src/main/protos/com/example/BUILD.bazel b/example/src/main/protos/com/example/BUILD.bazel new file mode 100644 index 00000000..5fe4fbd6 --- /dev/null +++ b/example/src/main/protos/com/example/BUILD.bazel @@ -0,0 +1,6 @@ +load('@rules_proto//proto:defs.bzl', 'proto_library') +load('@rules_python//python:proto.bzl', 'py_proto_library') +filegroup(name='example_files', srcs=glob(include=['**/*.proto']), visibility=['//visibility:public']) +proto_library(name='example', srcs=[':example_files'], deps=[], visibility=['//visibility:public']) +java_proto_library(name='example_java', deps=[':example'], visibility=['//visibility:public']) +py_proto_library(name='example_py', deps=[':example'], visibility=['//visibility:public']) \ No newline at end of file diff --git a/example/src/main/protos/com/example/foo/B.proto b/example/src/main/protos/com/example/foo/B.proto new file mode 100644 index 00000000..f90a68f8 --- /dev/null +++ b/example/src/main/protos/com/example/foo/B.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package com.example.foo; + +option java_multiple_files = true; + +import "src/main/protos/com/example/A.proto"; + +message B { + A a = 1; +} diff --git a/example/src/main/protos/com/example/foo/BUILD.bazel b/example/src/main/protos/com/example/foo/BUILD.bazel new file mode 100644 index 00000000..36488fec --- /dev/null +++ b/example/src/main/protos/com/example/foo/BUILD.bazel @@ -0,0 +1,6 @@ +load('@rules_proto//proto:defs.bzl', 'proto_library') +load('@rules_python//python:proto.bzl', 'py_proto_library') +filegroup(name='foo_files', srcs=glob(include=['**/*.proto']), visibility=['//visibility:public']) +proto_library(name='foo', srcs=[':foo_files'], deps=['//src/main/protos/com/example'], visibility=['//visibility:public']) +java_proto_library(name='foo_java', deps=[':foo'], visibility=['//visibility:public']) +py_proto_library(name='foo_py', deps=[':foo'], visibility=['//visibility:public']) \ No newline at end of file diff --git a/example/src/main/python/com/example/hello/BUILD.bazel b/example/src/main/python/com/example/hello/BUILD.bazel new file mode 100644 index 00000000..c005964c --- /dev/null +++ b/example/src/main/python/com/example/hello/BUILD.bazel @@ -0,0 +1,2 @@ +filegroup(name='hello_files', srcs=glob(include=['**/*.py']), visibility=['//visibility:public']) +py_library(name='hello', srcs=[':hello_files'], deps=['@@rules_python~0.24.0~pip~pip_39_pandas//:pkg'], visibility=['//visibility:public']) \ No newline at end of file diff --git a/example/src/main/python/com/example/hello/hello.py b/example/src/main/python/com/example/hello/hello.py new file mode 100644 index 00000000..cb93ae7d --- /dev/null +++ b/example/src/main/python/com/example/hello/hello.py @@ -0,0 +1,10 @@ +import pandas as pd + +def some_data(): + return pd.DataFrame() + +def main(): + print("Hello World!") + +if __name__ == "__main__": + main() diff --git a/example/src/test/java/com/example/BUILD.bazel b/example/src/test/java/com/example/BUILD.bazel new file mode 100644 index 00000000..12c359d6 --- /dev/null +++ b/example/src/test/java/com/example/BUILD.bazel @@ -0,0 +1,8 @@ +java_test( + name = "Test1", + srcs = ["Test1.java"], + deps = [ + "//src/main/protos/com/example:example_java", + "//src/main/protos/com/example/foo:foo_java", + ], +) diff --git a/example/src/test/java/com/example/Test1.java b/example/src/test/java/com/example/Test1.java new file mode 100644 index 00000000..968542a0 --- /dev/null +++ b/example/src/test/java/com/example/Test1.java @@ -0,0 +1,17 @@ +package com.example; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import com.example.A; +import com.example.foo.B; + +public class Test1 { + @Test + public void testA() throws Exception { + A a = A.newBuilder() + .addTags("foo") + .build(); + assertEquals(a.getTags(0), "foo"); + } +} diff --git a/example/src/test/python/com/example/hello/BUILD.bazel b/example/src/test/python/com/example/hello/BUILD.bazel new file mode 100644 index 00000000..4f132228 --- /dev/null +++ b/example/src/test/python/com/example/hello/BUILD.bazel @@ -0,0 +1 @@ +py_test(name='hello', srcs=glob(include=['**/*.py']), deps=['//src/main/python/com/example/hello'], visibility=['//visibility:public']) \ No newline at end of file diff --git a/example/src/test/python/com/example/hello/hello.py b/example/src/test/python/com/example/hello/hello.py new file mode 100644 index 00000000..a5f98b67 --- /dev/null +++ b/example/src/test/python/com/example/hello/hello.py @@ -0,0 +1,10 @@ +import unittest + +from src.main.python.com.example.hello.hello import some_data + +class HelloTest(unittest.TestCase): + def test_empty(self): + self.assertEqual(some_data().size, 0) + +if __name__ == '__main__': + unittest.main() diff --git a/prepare_all_apps.sh b/prepare_all_apps.sh index 95258726..547dbbde 100755 --- a/prepare_all_apps.sh +++ b/prepare_all_apps.sh @@ -26,7 +26,7 @@ if [ ! -d "$PREPARE_ALL_OUTPUT_DIR" ]; then mkdir -p $PREPARE_ALL_OUTPUT_DIR fi -echo "Running scala and python generator building" +echo "running Scala and Python generator building" OUTPUT_DIR=$PREPARE_ALL_OUTPUT_DIR language_generators/scala-defref-extractor/build_native.sh cd crates @@ -45,4 +45,4 @@ exports_files(["WORKSPACE", "python-entity-extractor"]) EOF -echo "Wrote all outputs to $PREPARE_ALL_OUTPUT_DIR" 1>&2 \ No newline at end of file +echo "wrote all outputs to $PREPARE_ALL_OUTPUT_DIR" 1>&2 From 456bc96c601bc1045345c950f566dd3df6e279c0 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 25 Jul 2023 15:22:13 -0400 Subject: [PATCH 2/3] use and_then --- crates/python_extractor/src/main.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/python_extractor/src/main.rs b/crates/python_extractor/src/main.rs index e82ed090..a52078f0 100644 --- a/crates/python_extractor/src/main.rs +++ b/crates/python_extractor/src/main.rs @@ -123,11 +123,10 @@ async fn main() -> Result<()> { } fn expand_path_to_defs_from_offset(from_given_path: &str, path: &str) -> Vec { - if let Some(rem0) = path.strip_prefix(from_given_path) { - let rem = match rem0.strip_prefix("site-packages/") { - Some(x) => x, - None => rem0 - }; + // rules_python Bzlmod support uses pip-tools, which I think places the 3rdparty + // source files inside a site-packages/ directory, per module. + if let Some(rem) = path.strip_prefix(from_given_path) + .and_then(|p| Some(p.strip_prefix("site-packages/").unwrap_or(p))) { if let Some(e) = rem.strip_suffix(".py") { let targ = e.replace('/', "."); From cdec70c70f4aab9610c547ffd076de0a15a3d92d Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 25 Jul 2023 16:22:01 -0400 Subject: [PATCH 3/3] Check that git diff is clean after regenerate --- .github/ci_scripts/integration_test.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/ci_scripts/integration_test.sh b/.github/ci_scripts/integration_test.sh index b96b1900..91f0015a 100755 --- a/.github/ci_scripts/integration_test.sh +++ b/.github/ci_scripts/integration_test.sh @@ -8,3 +8,10 @@ cd example # TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh bazel test ... + +changes=$(git diff --name-only --diff-filter=ACMRT | xargs) +if [ ! -z "$changes" ]; then + echo "::error file=example/WORKSPACE::Generated $changes differs from the checked-in version" + git diff --exit-code + exit 1 +fi