Example repo

.github/ci_scripts/integration_test.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+
+cd example
+# TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh
+TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh
+bazel test ...
+
+changes=$(git diff --name-only --diff-filter=ACMRT | xargs)
+if [ ! -z "$changes" ]; then
+    echo "::error file=example/WORKSPACE::Generated $changes differs from the checked-in version"
+    git diff --exit-code
+    exit 1
+fi

.github/workflows/test.yml

@@ -8,12 +8,22 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [macos-latest, ubuntu-latest]
         include:
           - os: macos-latest
             platform: macos
+            jobtype: 1
+          - os: macos-latest
+            platform: macos
+            jobtype: 3
+          - os: ubuntu-latest
+            platform: linux
+            jobtype: 1
           - os: ubuntu-latest
             platform: linux
+            jobtype: 2
+          - os: ubuntu-latest
+            platform: linux
+            jobtype: 3
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -33,7 +43,18 @@ jobs:
           distribution: temurin
           java-version: 8
           cache: sbt
-      - run: cd crates; cargo test --all-features
-      - run: cd crates; cargo test
-      - run: cd language_generators/scala-defref-extractor; ./sbt "test; scalafmtCheckAll"
-      - run: ./prepare_all_apps.sh
+      - name: Bazel cache
+        uses: actions/cache@v3
+        with:
+          path: "/home/runner/.cache/bazel"
+          key: bazel-${{ hashFiles('**/.bazelrc', '**/.bazeliskrc', '**/WORKSPACE', '**/WORKSPACE.bazel', '**/MODULE.bazel') }}
+      - if: ${{ matrix.jobtype == 1 }}
+        run: cd crates; cargo test --all-features
+      - if: ${{ matrix.jobtype == 1 }}
+        run: cd crates; cargo test
+      - if: ${{ matrix.jobtype == 2 }}
+        run: cd language_generators/scala-defref-extractor; ./sbt "test; scalafmtCheckAll"
+      - if: ${{ matrix.jobtype == 3 }}
+        run: |
+          ./prepare_all_apps.sh
+          .github/ci_scripts/integration_test.sh

.gitignore

@@ -7,3 +7,5 @@
 
 .vscode
 .metals
+
+bazel-*

crates/python_extractor/src/main.rs

@@ -123,12 +123,15 @@ async fn main() -> Result<()> {
 }
 
 fn expand_path_to_defs_from_offset(from_given_path: &str, path: &str) -> Vec<String> {
-    if let Some(rem) = path.strip_prefix(from_given_path) {
+    // rules_python Bzlmod support uses pip-tools, which I think places the 3rdparty
+    // source files inside a site-packages/ directory, per module.
+    if let Some(rem) = path.strip_prefix(from_given_path)
+        .and_then(|p| Some(p.strip_prefix("site-packages/").unwrap_or(p))) {
         if let Some(e) = rem.strip_suffix(".py") {
             let targ = e.replace('/', ".");
 
             if let Some(p) = targ.strip_suffix(".__init__") {
-                return vec![targ.clone(), p.to_string()];
+                return vec![p.to_string(), targ.clone()];
             } else {
                 return vec![targ];
             }
@@ -187,4 +190,16 @@ mod tests {
             expected
         );
     }
+
+    #[test]
+    fn expand_site_packages_path_to_defs_test() {
+        let mut expected = vec!["pytz", "pytz.__init__"];
+        expected.sort();
+        expected.dedup();
+
+        assert_eq!(
+            expand_path_to_defs_from_offset("/tmp/aaa/", "/tmp/aaa/site-packages/pytz/__init__.py"),
+            expected
+        );
+    }
 }

example/.bazelrc

@@ -0,0 +1 @@
+common --enable_bzlmod

example/.bazelversion

@@ -0,0 +1 @@
+6.2.1

example/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@pip//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement")
+load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test")
+load("@python_versions//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requirements")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+
+compile_pip_requirements_3_9(
+    name = "requirements_3_9",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_9.txt",
+)

example/MODULE.bazel

@@ -0,0 +1,20 @@
+module(
+    name = "example",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "bazel_skylib", version = "1.4.1")
+bazel_dep(name = "rules_python", version = "0.24.0")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    python_version = "3.9",
+)
+use_repo(python, "python_3_9", "python_versions")
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "pip",
+    requirements_lock = "//:requirements_lock_3_9.txt",
+)
+use_repo(pip, "pip", "pip_39")

example/WORKSPACE

@@ -0,0 +1,32 @@
+workspace(name = "example")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+skylib_version = "1.0.3"
+http_archive(
+    name = "bazel_skylib",
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
+    type = "tar.gz",
+    url = "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib-{}.tar.gz".format(skylib_version, skylib_version),
+)
+
+## Start: Protobuf
+http_archive(
+    name = "rules_proto",
+    url = "https://github.com/bazelbuild/rules_proto/archive/c0b62f2f46c85c16cb3b5e9e921f0d00e3101934.tar.gz",
+    sha256 = "e0cab008a9cdc2400a1d6572167bf9c5afc72e19ee2b862d18581051efab42c9",
+    strip_prefix = "rules_proto-c0b62f2f46c85c16cb3b5e9e921f0d00e3101934",
+)
+
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+
+rules_proto_dependencies()
+
+rules_proto_toolchains()
+
+## End: Protobuf
+
+local_repository(
+    name = "external_build_tooling_gen",
+    path = "build_tools/lang_support/create_lang_build_files/gen",
+)

example/build_tools/bazel_rules/wheel_scanner/BUILD.bazel

@@ -0,0 +1,5 @@
+py_binary(
+    name = "py_build_commands",
+    srcs = ["py_build_commands.py"],
+    visibility = ["//visibility:public"],
+)

example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py

@@ -0,0 +1,87 @@
+import sys
+import json
+import os
+import zipfile
+
+BATCH_SIZE = 2000
+
+PRELUDE = """
+#!/bin/bash
+
+set -efo pipefail
+
+set +x
+trap 'echo ERROR in ${BASH_SOURCE[0]}, failed to run command, line with error: $LINENO' ERR
+
+OUTPUT_BASE="$(bazel info output_base)"
+"""
+TEMPLATE = """
+
+echo -n "Running scan of 3rdparty files in batches, working on batch {output_idx}, with {target_count} targets in it"
+
+START_BATCH=$(date +%s)
+
+
+set +e
+bazel build {targets} \
+  --aspects build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl%wheel_scanner_aspect \
+  --output_groups=+wheel_scanner_out \
+  --override_repository=external_build_tooling_gen=${{BZL_GEN_BUILD_TOOLS_PATH}} \
+  --show_result=1000000 2> /tmp/cmd_out
+RET=$?
+if [ "$RET" != "0" ]; then
+    cat /tmp/cmd_out
+    exit $RET
+fi
+
+set +o pipefail
+inner_idx=0
+for f in `cat $OUTPUT_BASE/command.log |
+  grep ".*\.json$" |
+  sed -e 's/^[^ ]*//' |
+  sed -e 's/^[^A-Za-z0-9/]*//' |
+  sed 's/^ *//;s/ *$//'`; do
+  if [ -f "$f" ]; then
+    TARGET_PATH="${{BZL_BUILD_GEN_EXTERNAL_FILES_PATH}}/{output_idx}_${{inner_idx}}_wheel_scanner.json"
+    cp $f $TARGET_PATH
+    inner_idx=$((inner_idx + 1))
+  fi
+done
+
+set -o pipefail
+END_BATCH=$(date +%s)
+
+echo "...complete in $(($END_BATCH-$START_BATCH)) seconds"
+"""
+
+
+def __transform_target(t):
+    return "@%s//:pkg" % (t.lstrip("//external:"))
+
+def write_command(file, output_idx, command_list):
+    file.write(
+        TEMPLATE.format(
+            targets=" ".join([t for t in command_list if t.endswith("pkg")]),
+            output_idx=output_idx,
+            target_count=len(command_list),
+        )
+    )
+
+
+if __name__ == "__main__":
+    input_file = sys.argv[1]
+    output_file_path = sys.argv[2]
+    external_targets = []
+    output_idx = 0
+    with open(input_file, "r") as file1:
+        with open(output_file_path, "w") as output_file:
+            output_file.write(PRELUDE)
+            for line in file1.readlines():
+                external_targets.append(line.strip())
+                if len(external_targets) > BATCH_SIZE:
+                    write_command(output_file, output_idx, external_targets)
+                    output_idx += 1
+                    external_targets = []
+            if len(external_targets) > 0:
+                write_command(output_file, output_idx, external_targets)
+                output_idx += 1

example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl

@@ -0,0 +1,67 @@
+def _wheel_scanner_impl(target, ctx):
+    if not target.label.workspace_name.startswith("rules_python"):
+        return []
+
+    # Make sure the rule has a srcs attribute.
+    out_content = ctx.actions.declare_file("%s_wheel_scanner.json" % (target.label.name))
+
+    files = ctx.rule.files
+    all_py_files = []
+    all_py_relative_paths = []
+    workspace_root = target.label.workspace_root
+    if hasattr(files, "srcs"):
+        for file in files.srcs:
+            basename = file.basename
+
+            # We don't expect to have dependencies on tests
+            # These generated rules python files seem to have some invalid python syntax too
+            if not basename.startswith("rules_python_wheel_") and not basename.startswith("test_"):
+                path = file.path
+                if not path.startswith(workspace_root):
+                    fail("Didn't have workspace prefix")
+                all_py_relative_paths.append(path[len(workspace_root) + 1:])
+                all_py_files.append(file)
+
+    input_files = ctx.actions.declare_file("%s_wheel_scanner_input_files.txt" % (target.label.name))
+    ctx.actions.write(
+        input_files,
+        "\n".join(all_py_relative_paths),
+    )
+
+    args = ctx.actions.args()
+    args.add("--disable-ref-generation")
+    args.add("--label-or-repo-path")
+    args.add(str(target.label))
+
+    args.add("--import-path-relative-from")
+    args.add("%s/" % (target.label.workspace_root))
+    args.add("--working-directory")
+    args.add(target.label.workspace_root)
+    args.add("--relative-input-paths")
+    args.add("@%s" % input_files.path)
+    args.add("--output")
+    args.add(out_content)
+
+    inputs = [input_files]
+    inputs.extend(all_py_files)
+    ctx.actions.run(
+        outputs = [out_content],
+        inputs = inputs,
+        executable = ctx.files._py_exe[0],
+        mnemonic = "WheelScanner",
+        arguments = [args],
+    )
+
+    return [OutputGroupInfo(wheel_scanner_out = depset([out_content]))]
+
+wheel_scanner_aspect = aspect(
+    implementation = _wheel_scanner_impl,
+    attr_aspects = [],
+    attrs = {
+        "_py_exe": attr.label(
+            default = Label("@external_build_tooling_gen//:python-entity-extractor"),
+            allow_files = True,
+            cfg = "host",
+        ),
+    },
+)

example/build_tools/lang_support/create_lang_build_files/bazel_protos_modules.json

@@ -0,0 +1,48 @@
+{
+    "configurations": {
+      "protos": {
+        "file_extensions": [
+          "proto"
+        ],
+        "build_config": {
+          "main": {
+            "headers": [
+              {
+                "load_from": "@rules_proto//proto:defs.bzl",
+                "load_value": "proto_library"
+              }
+            ],
+            "function_name": "proto_library"
+          },
+          "secondary_rules": {
+            "java": {
+              "headers": [],
+              "function_name": "java_proto_library",
+              "extra_key_to_list": {
+                "deps": [":${name}"]
+              }
+            },
+            "py": {
+              "headers": [
+                {
+                  "load_from": "@rules_python//python:proto.bzl",
+                  "load_value": "py_proto_library"
+                }
+              ],
+              "function_name": "py_proto_library",
+              "extra_key_to_list": {
+                "deps": [":${name}"]
+              }
+            }
+          }
+        },
+        "main_roots": [
+          "src/main/protos"
+        ],
+        "test_roots": [
+          "src/test/protos"
+        ],
+        "path_directives": []
+      }
+    }
+}