Example repo (#89)

Problem
-------
In the current state bzl-gen-build is not really usable
outside of our own usage since it's relying on custom scripts and macros.
This is often necessitated by our tendency to use old version of Python etc.

Solution
--------
This adds an example repository to reimplement some of the
scripts and configurations using open source rules_python etc.

.github/ci_scripts/integration_test.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+
+cd example
+# TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_protos_build_files.sh
+TOOLING_WORKING_DIRECTORY=/tmp/bzl-gen-build source build_tools/lang_support/create_lang_build_files/regenerate_python_build_files.sh
+bazel test ...
+
+changes=$(git diff --name-only --diff-filter=ACMRT | xargs)
+if [ ! -z "$changes" ]; then
+    echo "::error file=example/WORKSPACE::Generated $changes differs from the checked-in version"
+    git diff --exit-code
+    exit 1
+fi

.github/workflows/test.yml

@@ -8,12 +8,22 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [macos-latest, ubuntu-latest]
         include:
           - os: macos-latest
             platform: macos
+            jobtype: 1
+          - os: macos-latest
+            platform: macos
+            jobtype: 3
+          - os: ubuntu-latest
+            platform: linux
+            jobtype: 1
           - os: ubuntu-latest
             platform: linux
+            jobtype: 2
+          - os: ubuntu-latest
+            platform: linux
+            jobtype: 3
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -33,7 +43,18 @@ jobs:
           distribution: temurin
           java-version: 8
           cache: sbt
-      - run: cd crates; cargo test --all-features
-      - run: cd crates; cargo test
-      - run: cd language_generators/scala-defref-extractor; ./sbt "test; scalafmtCheckAll"
-      - run: ./prepare_all_apps.sh
+      - name: Bazel cache
+        uses: actions/cache@v3
+        with:
+          path: "/home/runner/.cache/bazel"
+          key: bazel-${{ hashFiles('**/.bazelrc', '**/.bazeliskrc', '**/WORKSPACE', '**/WORKSPACE.bazel', '**/MODULE.bazel') }}
+      - if: ${{ matrix.jobtype == 1 }}
+        run: cd crates; cargo test --all-features
+      - if: ${{ matrix.jobtype == 1 }}
+        run: cd crates; cargo test
+      - if: ${{ matrix.jobtype == 2 }}
+        run: cd language_generators/scala-defref-extractor; ./sbt "test; scalafmtCheckAll"
+      - if: ${{ matrix.jobtype == 3 }}
+        run: |
+          ./prepare_all_apps.sh
+          .github/ci_scripts/integration_test.sh

.gitignore

@@ -7,3 +7,5 @@
 
 .vscode
 .metals
+
+bazel-*

crates/python_extractor/src/main.rs

@@ -123,12 +123,15 @@ async fn main() -> Result<()> {
 }
 
 fn expand_path_to_defs_from_offset(from_given_path: &str, path: &str) -> Vec<String> {
-    if let Some(rem) = path.strip_prefix(from_given_path) {
+    // rules_python Bzlmod support uses pip-tools, which I think places the 3rdparty
+    // source files inside a site-packages/ directory, per module.
+    if let Some(rem) = path.strip_prefix(from_given_path)
+        .and_then(|p| Some(p.strip_prefix("site-packages/").unwrap_or(p))) {
         if let Some(e) = rem.strip_suffix(".py") {
             let targ = e.replace('/', ".");
 
             if let Some(p) = targ.strip_suffix(".__init__") {
-                return vec![targ.clone(), p.to_string()];
+                return vec![p.to_string(), targ.clone()];
             } else {
                 return vec![targ];
             }
@@ -187,4 +190,16 @@ mod tests {
             expected
         );
     }
+
+    #[test]
+    fn expand_site_packages_path_to_defs_test() {
+        let mut expected = vec!["pytz", "pytz.__init__"];
+        expected.sort();
+        expected.dedup();
+
+        assert_eq!(
+            expand_path_to_defs_from_offset("/tmp/aaa/", "/tmp/aaa/site-packages/pytz/__init__.py"),
+            expected
+        );
+    }
 }

example/.bazelrc

@@ -0,0 +1 @@
+common --enable_bzlmod

example/.bazelversion

@@ -0,0 +1 @@
+6.2.1

example/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@pip//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement")
+load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test")
+load("@python_versions//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requirements")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+
+compile_pip_requirements_3_9(
+    name = "requirements_3_9",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_9.txt",
+)

example/MODULE.bazel

@@ -0,0 +1,20 @@
+module(
+    name = "example",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "bazel_skylib", version = "1.4.1")
+bazel_dep(name = "rules_python", version = "0.24.0")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    python_version = "3.9",
+)
+use_repo(python, "python_3_9", "python_versions")
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "pip",
+    requirements_lock = "//:requirements_lock_3_9.txt",
+)
+use_repo(pip, "pip", "pip_39")

example/WORKSPACE

@@ -0,0 +1,32 @@
+workspace(name = "example")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+skylib_version = "1.0.3"
+http_archive(
+    name = "bazel_skylib",
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
+    type = "tar.gz",
+    url = "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib-{}.tar.gz".format(skylib_version, skylib_version),
+)
+
+## Start: Protobuf
+http_archive(
+    name = "rules_proto",
+    url = "https://github.com/bazelbuild/rules_proto/archive/c0b62f2f46c85c16cb3b5e9e921f0d00e3101934.tar.gz",
+    sha256 = "e0cab008a9cdc2400a1d6572167bf9c5afc72e19ee2b862d18581051efab42c9",
+    strip_prefix = "rules_proto-c0b62f2f46c85c16cb3b5e9e921f0d00e3101934",
+)
+
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+
+rules_proto_dependencies()
+
+rules_proto_toolchains()
+
+## End: Protobuf
+
+local_repository(
+    name = "external_build_tooling_gen",
+    path = "build_tools/lang_support/create_lang_build_files/gen",
+)

example/build_tools/bazel_rules/wheel_scanner/BUILD.bazel

@@ -0,0 +1,5 @@
+py_binary(
+    name = "py_build_commands",
+    srcs = ["py_build_commands.py"],
+    visibility = ["//visibility:public"],
+)

example/build_tools/bazel_rules/wheel_scanner/py_build_commands.py

@@ -0,0 +1,87 @@
+import sys
+import json
+import os
+import zipfile
+
+BATCH_SIZE = 2000
+
+PRELUDE = """
+#!/bin/bash
+
+set -efo pipefail
+
+set +x
+trap 'echo ERROR in ${BASH_SOURCE[0]}, failed to run command, line with error: $LINENO' ERR
+
+OUTPUT_BASE="$(bazel info output_base)"
+"""
+TEMPLATE = """
+
+echo -n "Running scan of 3rdparty files in batches, working on batch {output_idx}, with {target_count} targets in it"
+
+START_BATCH=$(date +%s)
+
+
+set +e
+bazel build {targets} \
+  --aspects build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl%wheel_scanner_aspect \
+  --output_groups=+wheel_scanner_out \
+  --override_repository=external_build_tooling_gen=${{BZL_GEN_BUILD_TOOLS_PATH}} \
+  --show_result=1000000 2> /tmp/cmd_out
+RET=$?
+if [ "$RET" != "0" ]; then
+    cat /tmp/cmd_out
+    exit $RET
+fi
+
+set +o pipefail
+inner_idx=0
+for f in `cat $OUTPUT_BASE/command.log |
+  grep ".*\.json$" |
+  sed -e 's/^[^ ]*//' |
+  sed -e 's/^[^A-Za-z0-9/]*//' |
+  sed 's/^ *//;s/ *$//'`; do
+  if [ -f "$f" ]; then
+    TARGET_PATH="${{BZL_BUILD_GEN_EXTERNAL_FILES_PATH}}/{output_idx}_${{inner_idx}}_wheel_scanner.json"
+    cp $f $TARGET_PATH
+    inner_idx=$((inner_idx + 1))
+  fi
+done
+
+set -o pipefail
+END_BATCH=$(date +%s)
+
+echo "...complete in $(($END_BATCH-$START_BATCH)) seconds"
+"""
+
+
+def __transform_target(t):
+    return "@%s//:pkg" % (t.lstrip("//external:"))
+
+def write_command(file, output_idx, command_list):
+    file.write(
+        TEMPLATE.format(
+            targets=" ".join([t for t in command_list if t.endswith("pkg")]),
+            output_idx=output_idx,
+            target_count=len(command_list),
+        )
+    )
+
+
+if __name__ == "__main__":
+    input_file = sys.argv[1]
+    output_file_path = sys.argv[2]
+    external_targets = []
+    output_idx = 0
+    with open(input_file, "r") as file1:
+        with open(output_file_path, "w") as output_file:
+            output_file.write(PRELUDE)
+            for line in file1.readlines():
+                external_targets.append(line.strip())
+                if len(external_targets) > BATCH_SIZE:
+                    write_command(output_file, output_idx, external_targets)
+                    output_idx += 1
+                    external_targets = []
+            if len(external_targets) > 0:
+                write_command(output_file, output_idx, external_targets)
+                output_idx += 1

example/build_tools/bazel_rules/wheel_scanner/wheel_scanner.bzl

@@ -0,0 +1,67 @@
+def _wheel_scanner_impl(target, ctx):
+    if not target.label.workspace_name.startswith("rules_python"):
+        return []
+
+    # Make sure the rule has a srcs attribute.
+    out_content = ctx.actions.declare_file("%s_wheel_scanner.json" % (target.label.name))
+
+    files = ctx.rule.files
+    all_py_files = []
+    all_py_relative_paths = []
+    workspace_root = target.label.workspace_root
+    if hasattr(files, "srcs"):
+        for file in files.srcs:
+            basename = file.basename
+
+            # We don't expect to have dependencies on tests
+            # These generated rules python files seem to have some invalid python syntax too
+            if not basename.startswith("rules_python_wheel_") and not basename.startswith("test_"):
+                path = file.path
+                if not path.startswith(workspace_root):
+                    fail("Didn't have workspace prefix")
+                all_py_relative_paths.append(path[len(workspace_root) + 1:])
+                all_py_files.append(file)
+
+    input_files = ctx.actions.declare_file("%s_wheel_scanner_input_files.txt" % (target.label.name))
+    ctx.actions.write(
+        input_files,
+        "\n".join(all_py_relative_paths),
+    )
+
+    args = ctx.actions.args()
+    args.add("--disable-ref-generation")
+    args.add("--label-or-repo-path")
+    args.add(str(target.label))
+
+    args.add("--import-path-relative-from")
+    args.add("%s/" % (target.label.workspace_root))
+    args.add("--working-directory")
+    args.add(target.label.workspace_root)
+    args.add("--relative-input-paths")
+    args.add("@%s" % input_files.path)
+    args.add("--output")
+    args.add(out_content)
+
+    inputs = [input_files]
+    inputs.extend(all_py_files)
+    ctx.actions.run(
+        outputs = [out_content],
+        inputs = inputs,
+        executable = ctx.files._py_exe[0],
+        mnemonic = "WheelScanner",
+        arguments = [args],
+    )
+
+    return [OutputGroupInfo(wheel_scanner_out = depset([out_content]))]
+
+wheel_scanner_aspect = aspect(
+    implementation = _wheel_scanner_impl,
+    attr_aspects = [],
+    attrs = {
+        "_py_exe": attr.label(
+            default = Label("@external_build_tooling_gen//:python-entity-extractor"),
+            allow_files = True,
+            cfg = "host",
+        ),
+    },
+)

example/build_tools/lang_support/create_lang_build_files/bazel_protos_modules.json

@@ -0,0 +1,48 @@
+{
+    "configurations": {
+      "protos": {
+        "file_extensions": [
+          "proto"
+        ],
+        "build_config": {
+          "main": {
+            "headers": [
+              {
+                "load_from": "@rules_proto//proto:defs.bzl",
+                "load_value": "proto_library"
+              }
+            ],
+            "function_name": "proto_library"
+          },
+          "secondary_rules": {
+            "java": {
+              "headers": [],
+              "function_name": "java_proto_library",
+              "extra_key_to_list": {
+                "deps": [":${name}"]
+              }
+            },
+            "py": {
+              "headers": [
+                {
+                  "load_from": "@rules_python//python:proto.bzl",
+                  "load_value": "py_proto_library"
+                }
+              ],
+              "function_name": "py_proto_library",
+              "extra_key_to_list": {
+                "deps": [":${name}"]
+              }
+            }
+          }
+        },
+        "main_roots": [
+          "src/main/protos"
+        ],
+        "test_roots": [
+          "src/test/protos"
+        ],
+        "path_directives": []
+      }
+    }
+}