Merge pull request #10 from eed3si9n/wip/container_binary

container_binary

.mailmap

@@ -0,0 +1 @@
+Ian O'Connell <[email protected]> <[email protected]> 

minidock/BUILD.bazel

@@ -0,0 +1,5 @@
+filegroup(
+    name = "app_launch_template",
+    srcs = ["app_launch_template.txt"],
+    visibility = ["//visibility:public"],
+)

minidock/app_launch_template.txt

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+test -f /etc/profile.d/netflix_environment.sh && source /etc/profile.d/netflix_environment.sh
+
+export RUNFILES_DIR="%runfiles_path%"
+export JAVA_RUNFILES="%runfiles_path%"
+export JAVABIN=/usr/bin/java
+# For some reason the user doesn't seem to be set
+export USER="${USER:-$(whoami)}"
+exec %exec_path% "$@"

minidock/container_binary.bzl

@@ -0,0 +1,157 @@
+load(
+    "@com_github_bazeltools_rules_minidock//minidock:container_data.bzl",
+    "container_data",
+)
+load(
+    "@com_github_bazeltools_rules_minidock//minidock:container_compose.bzl",
+    "container_compose",
+)
+
+GROUP_COUNT = 15
+
+def __binary_to_outputs_impl(ctx):
+    group_idx = ctx.attr.group_idx
+
+    default_info = ctx.attr.binary[DefaultInfo]
+    base_files = default_info.default_runfiles.files.to_list()
+    if default_info.files != None:
+        base_files.extend(default_info.files.to_list())
+
+    if default_info.files_to_run != None:
+        files_to_run = default_info.files_to_run
+        if files_to_run.runfiles_manifest != None:
+            base_files.append(files_to_run.runfiles_manifest)
+
+    no_jdk = [x for x in base_files if x not in ctx.files._jdk]
+
+    def _shard(x):
+        p = x.short_path
+        if not p.startswith("../"):
+            return 0
+        hash_str = p.split("/")[1]
+        hash_v = hash(hash_str)
+        if hash_v < 0:
+            hash_v = hash_v * -1
+        hash_v = (hash_v % (GROUP_COUNT - 1)) + 1
+        return hash_v
+
+    filtered = depset([x for x in no_jdk if _shard(x) == group_idx])
+
+    return [DefaultInfo(files = filtered)]
+
+__binary_to_outputs = rule(
+    attrs = {
+        "binary": attr.label(mandatory = True, executable = True, cfg = "target"),
+        "group_idx": attr.int(),
+        "_jdk": attr.label(
+            default = Label("@bazel_tools//tools/jdk:current_java_runtime"),
+            providers = [java_common.JavaRuntimeInfo],
+        ),
+    },
+    implementation = __binary_to_outputs_impl,
+)
+
+def _launcher_script_impl(ctx):
+    template = ctx.attr.app_launch_template.files.to_list()[0]
+    exe = ctx.actions.declare_file(
+        "%s.files/%s" % (ctx.attr.name, ctx.attr.exe_name),
+    )
+
+    ctx.actions.expand_template(
+        template = template,
+        output = exe,
+        substitutions = {
+            "%runfiles_path%": ctx.attr.runfiles_path,
+            "%exec_path%": ctx.attr.exec_path_path,
+        },
+        is_executable = True,
+    )
+
+    return [
+        DefaultInfo(files = depset([exe])),
+    ]
+
+launcher_script = rule(
+    attrs = {
+        "exe_name": attr.string(),
+        "exec_path_path": attr.string(),
+        "runfiles_path": attr.string(),
+        "app_launch_template": attr.label(),
+    },
+    implementation = _launcher_script_impl,
+)
+
+def _binary_data_layers(name, jvm_binary):
+    results = []
+    for idx in range(0, GROUP_COUNT):
+        merge_transformer = "%s.data.transformer%d" % (name, idx)
+        layer_data = "%s.data.layer.external.%d" % (name, idx)
+        if idx == 0:
+            layer_data = "%s.data.layer.%d" % (name, idx)
+        __binary_to_outputs(name = merge_transformer, binary = jvm_binary, group_idx = idx)
+        container_data(
+            name = layer_data,
+            files = [merge_transformer],
+            gzip_compression_level = 0,
+            data_path = "/",
+            directory = "/usr/local/data/%s.runfiles/minidock" % (name),
+            tags = ["expensive_build", "no-remote-cache"],
+        )
+        results.append(layer_data)
+    return results
+
+def container_binary(
+        name,
+        binary,
+        binary_name = None,
+        app_launch_template_script = "@com_github_bazeltools_rules_minidock//minidock:app_launch_template"):
+    if binary_name == None:
+        binary_name = name
+    launcher_name = "%s.launcher" % name
+    launcher_data = "%s.launcher.data" % name
+
+    path = binary
+    if path.startswith("//"):
+        path = path.lstrip("//")
+    elif path.startswith(":") or not ("/" in path):
+        path = path.lstrip(":")
+        path = "%s/%s" % (native.package_name(), path)
+    else:
+        fail("Don't know how to handle path %s" % path)
+
+    path = path.replace(":", "/")
+
+    data_layers = _binary_data_layers(
+        name,
+        binary,
+    )
+
+    in_container_data_binary = "/usr/local/data/{target_name}.runfiles/minidock/{path}".format(
+        path = path,
+        target_name = name,
+    )
+
+    runfiles_path = "/usr/local/data/%s.runfiles" % name
+    launcher_script(
+        name = launcher_name,
+        exe_name = binary_name,
+        exec_path_path = in_container_data_binary,
+        runfiles_path = runfiles_path,
+        app_launch_template = app_launch_template_script,
+    )
+
+    container_data(
+        name = launcher_data,
+        files = [launcher_name],
+        gzip_compression_level = 0,
+        directory = "/usr/bin",
+        tags = ["expensive_build"],
+    )
+    container_compose(
+        name = name,
+        layers = data_layers + [
+            launcher_data,
+        ],
+        tags = ["expensive_build"],
+        visibility = ["//visibility:public"],
+    )

tests/simple_flow/.bazelrc

@@ -0,0 +1 @@
+common --enable_bzlmod

tests/simple_flow/BUILD.bazel

@@ -19,6 +19,28 @@ load(
     "@com_github_bazeltools_rules_minidock//minidock:container_config.bzl",
     "container_config",
 )
+load(
+    "@com_github_bazeltools_rules_minidock//minidock:container_binary.bzl",
+    "container_binary",
+)
+load(
+    "@python_versions//3.10:defs.bzl",
+    compile_pip_requirements_3_10 = "compile_pip_requirements",
+)
+load(
+    "@rules_python//python:defs.bzl",
+    "py_binary",
+)
+
+PANDAS = "@@rules_python~0.24.0~pip~pip_310_pandas//:pkg"
+
+compile_pip_requirements_3_10(
+    name = "requirements_3_10",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_10.txt",
+)
+
 genrule(
     name = "external_config_file",
     outs = ["ConfigFile.json"],
@@ -104,3 +126,15 @@ container_push(
     repository = "docker/test",
     container_tags = ["unchanged_tag1"],
 )
+
+py_binary(
+    name = "hello",
+    srcs = ["hello.py"],
+    deps = [PANDAS],
+    visibility = ["//visibility:public"],
+)
+
+container_binary(
+    name = "bin",
+    binary = ":hello",
+)

tests/simple_flow/MODULE.bazel

@@ -0,0 +1,20 @@
+module(
+    name = "example",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "bazel_skylib", version = "1.4.1")
+bazel_dep(name = "rules_python", version = "0.24.0")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    python_version = "3.10",
+)
+use_repo(python, "python_3_10", "python_versions")
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "pip",
+    requirements_lock = "//:requirements_lock_3_10.txt",
+)
+use_repo(pip, "pip", "pip_310")

tests/simple_flow/hello.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+FOO = [""]
+
+def some_data():
+    return pd.DataFrame()
+
+def main():
+    print("Hello World!")
+
+if __name__ == "__main__":
+    main()

tests/simple_flow/requirements.in

@@ -0,0 +1,4 @@
+--extra-index-url https://pypi.python.org/simple/
+
+protobuf
+pandas

tests/simple_flow/requirements_lock_3_10.txt

@@ -0,0 +1,106 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    bazel run //:requirements_3_10.update
+#
+--extra-index-url https://pypi.python.org/simple/
+
+numpy==1.26.3 \
+    --hash=sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd \
+    --hash=sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b \
+    --hash=sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e \
+    --hash=sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f \
+    --hash=sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f \
+    --hash=sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178 \
+    --hash=sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3 \
+    --hash=sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4 \
+    --hash=sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e \
+    --hash=sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0 \
+    --hash=sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00 \
+    --hash=sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419 \
+    --hash=sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4 \
+    --hash=sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6 \
+    --hash=sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166 \
+    --hash=sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b \
+    --hash=sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3 \
+    --hash=sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf \
+    --hash=sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2 \
+    --hash=sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2 \
+    --hash=sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36 \
+    --hash=sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03 \
+    --hash=sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce \
+    --hash=sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6 \
+    --hash=sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13 \
+    --hash=sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5 \
+    --hash=sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e \
+    --hash=sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485 \
+    --hash=sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137 \
+    --hash=sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374 \
+    --hash=sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58 \
+    --hash=sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b \
+    --hash=sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb \
+    --hash=sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b \
+    --hash=sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda \
+    --hash=sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511
+    # via pandas
+pandas==2.2.0 \
+    --hash=sha256:159205c99d7a5ce89ecfc37cb08ed179de7783737cea403b295b5eda8e9c56d1 \
+    --hash=sha256:20404d2adefe92aed3b38da41d0847a143a09be982a31b85bc7dd565bdba0f4e \
+    --hash=sha256:2707514a7bec41a4ab81f2ccce8b382961a29fbe9492eab1305bb075b2b1ff4f \
+    --hash=sha256:30b83f7c3eb217fb4d1b494a57a2fda5444f17834f5df2de6b2ffff68dc3c8e2 \
+    --hash=sha256:38e0b4fc3ddceb56ec8a287313bc22abe17ab0eb184069f08fc6a9352a769b18 \
+    --hash=sha256:3de918a754bbf2da2381e8a3dcc45eede8cd7775b047b923f9006d5f876802ae \
+    --hash=sha256:52826b5f4ed658fa2b729264d63f6732b8b29949c7fd234510d57c61dbeadfcd \
+    --hash=sha256:57abcaeda83fb80d447f28ab0cc7b32b13978f6f733875ebd1ed14f8fbc0f4ab \
+    --hash=sha256:5a946f210383c7e6d16312d30b238fd508d80d927014f3b33fb5b15c2f895430 \
+    --hash=sha256:736da9ad4033aeab51d067fc3bd69a0ba36f5a60f66a527b3d72e2030e63280a \
+    --hash=sha256:761cb99b42a69005dec2b08854fb1d4888fdf7b05db23a8c5a099e4b886a2106 \
+    --hash=sha256:7ea3ee3f125032bfcade3a4cf85131ed064b4f8dd23e5ce6fa16473e48ebcaf5 \
+    --hash=sha256:8108ee1712bb4fa2c16981fba7e68b3f6ea330277f5ca34fa8d557e986a11670 \
+    --hash=sha256:85793cbdc2d5bc32620dc8ffa715423f0c680dacacf55056ba13454a5be5de88 \
+    --hash=sha256:8ce2fbc8d9bf303ce54a476116165220a1fedf15985b09656b4b4275300e920b \
+    --hash=sha256:9f66419d4a41132eb7e9a73dcec9486cf5019f52d90dd35547af11bc58f8637d \
+    --hash=sha256:a146b9dcacc3123aa2b399df1a284de5f46287a4ab4fbfc237eac98a92ebcb71 \
+    --hash=sha256:a1b438fa26b208005c997e78672f1aa8138f67002e833312e6230f3e57fa87d5 \
+    --hash=sha256:a20628faaf444da122b2a64b1e5360cde100ee6283ae8effa0d8745153809a2e \
+    --hash=sha256:a41d06f308a024981dcaa6c41f2f2be46a6b186b902c94c2674e8cb5c42985bc \
+    --hash=sha256:a626795722d893ed6aacb64d2401d017ddc8a2341b49e0384ab9bf7112bdec30 \
+    --hash=sha256:bde2bc699dbd80d7bc7f9cab1e23a95c4375de615860ca089f34e7c64f4a8de7 \
+    --hash=sha256:cfd6c2491dc821b10c716ad6776e7ab311f7df5d16038d0b7458bc0b67dc10f3 \
+    --hash=sha256:e60f1f7dba3c2d5ca159e18c46a34e7ca7247a73b5dd1a22b6d59707ed6b899a \
+    --hash=sha256:eb1e1f3861ea9132b32f2133788f3b14911b68102d562715d71bd0013bc45440 \
+    --hash=sha256:eb61dc8567b798b969bcc1fc964788f5a68214d333cade8319c7ab33e2b5d88a \
+    --hash=sha256:f5be5d03ea2073627e7111f61b9f1f0d9625dc3c4d8dda72cc827b0c58a1d042 \
+    --hash=sha256:f9670b3ac00a387620489dfc1bca66db47a787f4e55911f1293063a78b108df1 \
+    --hash=sha256:fbc1b53c0e1fdf16388c33c3cca160f798d38aea2978004dd3f4d3dec56454c9
+    # via -r requirements.in
+protobuf==4.25.2 \
+    --hash=sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62 \
+    --hash=sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d \
+    --hash=sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61 \
+    --hash=sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62 \
+    --hash=sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3 \
+    --hash=sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9 \
+    --hash=sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830 \
+    --hash=sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6 \
+    --hash=sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0 \
+    --hash=sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020 \
+    --hash=sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e
+    # via -r requirements.in
+python-dateutil==2.8.2 \
+    --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
+    --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
+    # via pandas
+pytz==2023.3.post1 \
+    --hash=sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b \
+    --hash=sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7
+    # via pandas
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+    # via python-dateutil
+tzdata==2023.4 \
+    --hash=sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3 \
+    --hash=sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9
+    # via pandas