diff --git a/README.md b/README.md index 6b92dc6..ffb2958 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,43 @@ You can use the rules directly from this repo, or simply as examples and copy/paste them into your own project. +Given a pycross_oci_image rule `//my:image` wrapping a py_binary rule `//my:app`, the following targets are defined: + +| Kind | Label | Description | +|---------------------------------|------------------------------|----------------------------------------------------------| +| oci_image rule | //my:image | The target container image | +| oci_tarball rule | //my:image.tar | Tarball rule that can be `bazel run` to `docker load` it | +| tar rule | //my:image.app_layer | Image layer for the application code | +| tar rule | //my:image.packages_layer | Image layer for site-packages | +| tar rule | //my:image.interpreter_layer | Image layer for the python3 interpreter | +| platform_transition_binary rule | //my:xapp | The transitioned "cross" py_binary | +| py_binary rule | //my:app | The source py_binary app | + +`docker inspect` yields the following (subset): + +```json +[ + { + "RepoTags": [ + "my/app:latest", + ], + "Config": { + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt", + "LANG=C.UTF-8" + ], + "Cmd": [ + "app" + ], + "WorkingDir": "/my/xapp", + "Entrypoint": [ + "/my/xapp/app.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" + ], + }, + "Architecture": "amd64", + "Os": "linux" + } +] +``` + diff --git a/WORKSPACE b/WORKSPACE index fa5544b..f9811ec 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -26,13 +26,15 @@ step1.setup_rules_oci() step1.setup_rules_docker() +step1.setup_container_structure_test() + # ----------------------------------------- load("@pycross_image//bazel/workspace:step2.bzl", "step2") -step2.setup_oci_containers() +step2.setup_pycross_image_base_oci() -step2.setup_docker_containers() +step2.setup_pycross_image_base_container() step2.setup_rules_pycross() diff --git a/bazel/rules/BUILD.bazel b/bazel/rules/BUILD.bazel index 0c1502b..4dd55d7 100644 --- a/bazel/rules/BUILD.bazel +++ b/bazel/rules/BUILD.bazel @@ -6,6 +6,6 @@ example_test_filegroup( "BUILD.bazel", "docker.bzl", "oci.bzl", - "py_layers.bzl", + "support.bzl", ], ) diff --git a/bazel/rules/docker.bzl b/bazel/rules/docker.bzl index 5a235eb..bcb111c 100644 --- a/bazel/rules/docker.bzl +++ b/bazel/rules/docker.bzl @@ -4,107 +4,50 @@ Wrapper macro to make three separate layers for python applications based on: https://github.com/aspect-build/bazel-examples/blob/a25b6c0ba307545aff6c4b5feb4ae875d7d507f1/oci_python_image/py_layer.bzl """ -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_binary") -load(":py_layers.bzl", "py_layers") +load(":support.bzl", "defaults", "py_layers", "pycross_binary") load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_layer") -def _make_entrypoint(toolchain_config_setting_label, workdir, cmd): - label = Label(toolchain_config_setting_label) - return "{workdir}/{cmd}.runfiles/{python_toolchains_workspace_name}_{python_toolchains_config_setting}/bin/python3".format( - workdir = workdir, - cmd = cmd, - python_toolchains_workspace_name = label.workspace_name, - python_toolchains_config_setting = label.name, - ) - def py_image( name, binary, - base = "@distroless_python3_debian12_container//image", - config = "@python//:x86_64-unknown-linux-gnu", - target_platform = "@pycross_image//bazel/platforms:linux_x86_64", + base = "@pycross_image_base_container//image", + layers = [], **kwargs): """ - pycross_oci_image is a macro that instantiates an oci_image from a py_binary rule - - Given a pycross_oci_image rule `//my:image` wrapping a py_binary rule `//my:app`, the following targets are defined: - - | Kind | Label | Description | - |---------------------------------|------------------------------|----------------------------------------------------------| - | oci_image rule | //my:image | The target container image | - | oci_tarball rule | //my:image.tar | Tarball rule that can be `bazel run` to `docker load` it | - | tar rule | //my:image.app_layer | Image layer for the application code | - | tar rule | //my:image.packages_layer | Image layer for site-packages | - | tar rule | //my:image.interpreter_layer | Image layer for the python3 interpreter | - | platform_transition_binary rule | //my:xapp | The transitioned "cross" py_binary | - | py_binary rule | //my:app | The source py_binary app | - - `docker inspect` yields the following (subset): - - ```json - [ - { - "RepoTags": [ - "my/app:latest", - ], - "Config": { - "Env": [ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt", - "LANG=C.UTF-8" - ], - "Cmd": [ - "app" - ], - "WorkingDir": "/my/xapp", - "Entrypoint": [ - "/my/xapp/app.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" - ], - }, - "Architecture": "amd64", - "Os": "linux" - } - ] - ``` + py_image is a macro that instantiates an container_image from a py_binary rule Args: - name: (String) name for the oci_image rule - binary: (Label) target label of the py_binary rule - base: (Label) target label of the base image - tars: (Label List) optional additional tars for the image - config: (Label) target label for the python toolchains repo config setting that determines the python entrypoint name - target_platform: (Label) target label for the image platform (for the platform_transition_binary rule) - **kwargs: (Dict) additional argument for the oci_image rule + name: name for the image rule + binary: target label of the py_binary rule + base: target label of the base image + layers: additional container_layer targets + **kwargs: additional arguments for the pycross_binary and container_image rules """ - target = Label(binary) - cmd = target.name - cross_binary = cmd + "_cross_binary" - repo_tag = "%s/%s:latest" % (target.package, target.name) - - workdir = "/%s/%s/%s" % (target.package, target.name, cross_binary) - entrypoint = _make_entrypoint(config, workdir, cmd) - - layer_tars = py_layers(name, cross_binary) - - platform_transition_binary( - name = cross_binary, - binary = binary, - target_platform = target_platform, + metadata = pycross_binary( + binary, + cross_name_prefix = kwargs.pop("cross_name_prefix", defaults.cross_name_prefix), + python_config_setting = kwargs.pop("python_config_setting", defaults.python_config_setting), + target_platform = kwargs.pop("target_platform", defaults.target_platform), + image_tag = kwargs.pop("tag", defaults.cross_name_prefix), + repo_tag = kwargs.pop("repo_tag", None), ) - for tar in layer_tars: + tars = py_layers(name, metadata.cross_binary_name) + + for tar in tars: container_layer( name = tar + "_layer", tars = [tar], ) container_image( - name = name + "_container", + name = name, base = base, - layers = [tar + "_layer" for tar in layer_tars], - workdir = workdir, - entrypoint = [entrypoint], - tags = [repo_tag], - cmd = [cmd], + layers = layers + [tar + "_layer" for tar in tars], + entrypoint = metadata.entrypoint, + cmd = metadata.cmd, + workdir = metadata.workdir, + tags = [metadata.repo_tag], + **kwargs ) diff --git a/bazel/rules/oci.bzl b/bazel/rules/oci.bzl index fbda182..e6a6238 100644 --- a/bazel/rules/oci.bzl +++ b/bazel/rules/oci.bzl @@ -1,112 +1,45 @@ -""" -Wrapper macro to make three separate layers for python applications - -based on: https://github.com/aspect-build/bazel-examples/blob/a25b6c0ba307545aff6c4b5feb4ae875d7d507f1/oci_python_image/py_layer.bzl -""" - -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_binary") +load(":support.bzl", "defaults", "py_layers", "pycross_binary") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball") -load(":py_layers.bzl", "py_layers") - -def _make_entrypoint(toolchain_config_setting_label, workdir, cmd): - label = Label(toolchain_config_setting_label) - return "{workdir}/{cmd}.runfiles/{python_toolchains_workspace_name}_{python_toolchains_config_setting}/bin/python3".format( - workdir = workdir, - cmd = cmd, - python_toolchains_workspace_name = label.workspace_name, - python_toolchains_config_setting = label.name, - ) def py_image( name, binary, - base = "@distroless_python3_debian12_oci", - tars = [], - config = "@python//:x86_64-unknown-linux-gnu", - target_platform = "@pycross_image//bazel/platforms:linux_x86_64", + base = "@pycross_image_base_oci", + layers = [], **kwargs): """ - pycross_oci_image is a macro that instantiates an oci_image from a py_binary rule - - Given a pycross_oci_image rule `//my:image` wrapping a py_binary rule `//my:app`, the following targets are defined: - - | Kind | Label | Description | - |---------------------------------|------------------------------|----------------------------------------------------------| - | oci_image rule | //my:image | The target container image | - | oci_tarball rule | //my:image.tar | Tarball rule that can be `bazel run` to `docker load` it | - | tar rule | //my:image.app_layer | Image layer for the application code | - | tar rule | //my:image.packages_layer | Image layer for site-packages | - | tar rule | //my:image.interpreter_layer | Image layer for the python3 interpreter | - | platform_transition_binary rule | //my:xapp | The transitioned "cross" py_binary | - | py_binary rule | //my:app | The source py_binary app | - - `docker inspect` yields the following (subset): - - ```json - [ - { - "RepoTags": [ - "my/app:latest", - ], - "Config": { - "Env": [ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt", - "LANG=C.UTF-8" - ], - "Cmd": [ - "app" - ], - "WorkingDir": "/my/xapp", - "Entrypoint": [ - "/my/xapp/app.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" - ], - }, - "Architecture": "amd64", - "Os": "linux" - } - ] - ``` + py_image is a macro that instantiates an oci_image from a py_binary rule Args: - name: (String) name for the oci_image rule - binary: (Label) target label of the py_binary rule - base: (Label) target label of the base image - tars: (Label List) optional additional tars for the image - config: (Label) target label for the python toolchains repo config setting that determines the python entrypoint name - target_platform: (Label) target label for the image platform (for the platform_transition_binary rule) - **kwargs: (Dict) additional argument for the oci_image rule + name: name for the image rule + binary: target label of the py_binary rule + base: target label of the base image + layers: additional layer tarballs + **kwargs: additional arguments for the pycross_binary and oci_image rules """ - - name_tar = name + ".tar" - target = Label(binary) - cmd = target.name - cross_binary = cmd + "_cross_binary" - repo_tag = "%s/%s:latest" % (target.package, target.name) - - workdir = "/%s/%s/%s" % (target.package, target.name, cross_binary) - entrypoint = _make_entrypoint(config, workdir, cmd) - - layer_tars = py_layers(name, cross_binary) - - platform_transition_binary( - name = cross_binary, - binary = binary, - target_platform = target_platform, + metadata = pycross_binary( + binary, + cross_name_prefix = kwargs.pop("cross_name_prefix", "y"), + python_config_setting = kwargs.pop("python_config_setting", defaults.python_config_setting), + target_platform = kwargs.pop("target_platform", defaults.target_platform), + image_tag = kwargs.pop("tag", defaults.cross_name_prefix), + repo_tag = kwargs.pop("repo_tag", None), ) + tars = py_layers(name, metadata.cross_binary_name) + oci_image( name = name, - tars = tars + layer_tars, - workdir = workdir, - entrypoint = [entrypoint], - cmd = [cmd], base = base, + tars = layers + tars, + workdir = metadata.workdir, + entrypoint = metadata.entrypoint, + cmd = metadata.cmd, **kwargs ) oci_tarball( - name = name_tar, + name = name + ".tar", image = name, - repo_tags = [repo_tag], + repo_tags = [metadata.repo_tag], ) diff --git a/bazel/rules/py_layers.bzl b/bazel/rules/py_layers.bzl deleted file mode 100644 index 9b96eb4..0000000 --- a/bazel/rules/py_layers.bzl +++ /dev/null @@ -1,71 +0,0 @@ -""" -Wrapper macro to make three separate layers for python applications - -based on: https://github.com/aspect-build/bazel-examples/blob/a25b6c0ba307545aff6c4b5feb4ae875d7d507f1/oci_python_image/py_layer.bzl -""" - -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") - -def py_layers(name, binary): - """Create three layers for a py_binary target: interpreter, third-party dependencies, and application code. - - This allows a container image to have smaller uploads, since the application layer usually changes more - than the other two. - - Args: - name: prefix for generated targets, to ensure they are unique within the package - binary: a py_binary target - Returns: - a list of labels for the layers, which are tar files - """ - - # Produce the manifest for a tar file of our py_binary, but don't tar it up yet, so we can split - # into fine-grained layers for better docker performance. - mtree_spec( - name = name + ".mf", - srcs = [binary], - ) - - # match *only* external repositories that have the string "python" - # e.g. this will match - # `/hello_world/hello_world_bin.runfiles/rules_python~0.21.0~python~python3_9_aarch64-unknown-linux-gnu/bin/python3` - # but not match - # `/hello_world/hello_world_bin.runfiles/_main/python_app` - py_interpreter_regex = "\\.runfiles/.*python.*-.*" - - # match *only* external pip like repositories that contain the string "site-packages" - site_packages_regex = "\\.runfiles/.*/site-packages/.*" - - native.genrule( - name = name + ".interpreter_tar_manifest", - srcs = [name + ".mf"], - outs = [name + ".interpreter_tar_manifest.spec"], - cmd = "grep '{}' $< >$@".format(py_interpreter_regex), - ) - - native.genrule( - name = name + ".packages_tar_manifest", - srcs = [name + ".mf"], - outs = [name + ".packages_tar_manifest.spec"], - cmd = "grep '{}' $< >$@".format(site_packages_regex), - ) - - # Any lines that didn't match one of the two grep above - native.genrule( - name = name + ".app_tar_manifest", - srcs = [name + ".mf"], - outs = [name + ".app_tar_manifest.spec"], - cmd = "grep -v '{}' $< | grep -v '{}' >$@".format(site_packages_regex, py_interpreter_regex), - ) - - result = [] - for layer in ["interpreter", "packages", "app"]: - layer_target = "{}.{}_layer".format(name, layer) - result.append(layer_target) - tar( - name = layer_target, - srcs = [binary], - mtree = "{}.{}_tar_manifest".format(name, layer), - ) - - return result diff --git a/bazel/rules/support.bzl b/bazel/rules/support.bzl new file mode 100644 index 0000000..ce381f1 --- /dev/null +++ b/bazel/rules/support.bzl @@ -0,0 +1,143 @@ +""" +Support functions +""" + +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_binary") + +defaults = struct( + python_config_setting = "@python//:x86_64-unknown-linux-gnu", + target_platform = "@pycross_image//bazel/platforms:linux_x86_64", + cross_name_prefix = "x", + image_tag = "latest", +) + +def py_layers(name, binary): + """Create three layers for a py_binary target: interpreter, third-party dependencies, and application code. + + based on: https://github.com/aspect-build/bazel-examples/blob/a25b6c0ba307545aff6c4b5feb4ae875d7d507f1/oci_python_image/py_layer.bzl + + This allows a container image to have smaller uploads, since the application layer usually changes more + than the other two. + + Args: + name: prefix for generated targets, to ensure they are unique within the package + binary: a py_binary target + Returns: + a list of labels for the layers, which are tar files + """ + + # Produce the manifest for a tar file of our py_binary, but don't tar it up yet, so we can split + # into fine-grained layers for better docker performance. + mtree_spec( + name = name + ".mf", + srcs = [binary], + ) + + # match *only* external repositories that have the string "python" + # e.g. this will match + # `/hello_world/hello_world_bin.runfiles/rules_python~0.21.0~python~python3_9_aarch64-unknown-linux-gnu/bin/python3` + # but not match + # `/hello_world/hello_world_bin.runfiles/_main/python_app` + py_interpreter_regex = "\\.runfiles/.*python.*-.*" + + # match *only* external pip like repositories that contain the string "site-packages" + site_packages_regex = "\\.runfiles/.*/site-packages/.*" + + native.genrule( + name = name + ".interpreter_tar_manifest", + srcs = [name + ".mf"], + outs = [name + ".interpreter_tar_manifest.spec"], + cmd = "grep '{}' $< >$@".format(py_interpreter_regex), + ) + + native.genrule( + name = name + ".packages_tar_manifest", + srcs = [name + ".mf"], + outs = [name + ".packages_tar_manifest.spec"], + cmd = "grep '{}' $< >$@".format(site_packages_regex), + ) + + # Any lines that didn't match one of the two grep above + native.genrule( + name = name + ".app_tar_manifest", + srcs = [name + ".mf"], + outs = [name + ".app_tar_manifest.spec"], + cmd = "grep -v '{}' $< | grep -v '{}' >$@".format(site_packages_regex, py_interpreter_regex), + ) + + result = [] + for layer in ["interpreter", "packages", "app"]: + layer_target = "{}.{}_layer".format(name, layer) + result.append(layer_target) + tar( + name = layer_target, + srcs = [binary], + mtree = "{}.{}_tar_manifest".format(name, layer), + ) + + return result + +def _make_entrypoint(toolchain_config_setting_label, workdir, cmd): + label = Label(toolchain_config_setting_label) + return "{workdir}/{cmd}.runfiles/{python_toolchains_workspace_name}_{python_toolchains_config_setting}/bin/python3".format( + workdir = workdir, + cmd = cmd, + python_toolchains_workspace_name = label.workspace_name, + python_toolchains_config_setting = label.name, + ) + +def pycross_binary( + binary, + name = None, + cross_name_prefix = defaults.cross_name_prefix, + python_config_setting = defaults.python_config_setting, + target_platform = defaults.target_platform, + image_tag = defaults.image_tag, + repo_tag = None): + """pycross_binary declares a transitioned py_binary and prepares the image metadata. + + Args: + name: name of the cross_binary, will be + binary: a Label to a py_binary to be packaged + cross_name_prefix: a prefix that can be used to make the platform_transition_binary name more unique. + python_config_setting: optional label to the python interpreter config_setting rule. + target_platform: Label string: the platform to transition to + image_tag: a tag for the image, defaults to "latest" + repo_tag: repo_tag for the image, defaults to '{package_name}/{label_name}/{tag}' + Returns: + a struct for the image metadata. + """ + + # Label: the label of the py_binary to be packaged + target = native.package_relative_label(binary) + + # string: a label name for the platform_transition_binary. use a prefix such + # that if a package has both an oci and docker py_image, the + # platform_transition_binary names don't collide. + name = cross_name_prefix + target.name + + # string: the workdir. The entrypoint will be in the directory where the + # platform_transition_binary lives + workdir = "/%s/%s" % (target.package, name) + + # string: the (repo_)tag of the container. Can be overridden by user. + if not repo_tag: + repo_tag = "%s/%s:%s" % (target.package, target.name, image_tag) + + # string: the entrypoint. See structure test for details. + entrypoint = _make_entrypoint(python_config_setting, workdir, target.name) + + platform_transition_binary( + name = name, + binary = binary, + target_platform = target_platform, + ) + + return struct( + cross_binary_name = name, + workdir = workdir, + entrypoint = [entrypoint], + cmd = [target.name], + repo_tag = repo_tag, + ) diff --git a/bazel/tools/pdm/BUILD.bazel b/bazel/tools/pdm/BUILD.bazel index 7154758..d907542 100644 --- a/bazel/tools/pdm/BUILD.bazel +++ b/bazel/tools/pdm/BUILD.bazel @@ -1,5 +1,8 @@ load("@rules_python//python:defs.bzl", "py_binary") +load("@io_bazel_rules_docker//contrib:test.bzl", "container_test") +load("@container_structure_test//:defs.bzl", image_test = "container_structure_test") load("@//bazel/rules:oci.bzl", "py_image") +load("@//bazel/rules:docker.bzl", py_container = "py_image") py_binary( name = "pdm", @@ -8,13 +11,24 @@ py_binary( deps = ["@pypi_deps_for_pdm//:pdm"], ) -sh_binary( - name = "run", - srcs = ["run.sh"], - data = [":pdm"], -) - py_image( name = "image", binary = ":pdm", ) + +image_test( + name = "image_test", + configs = ["image-test.yaml"], + image = ":image", +) + +py_container( + name = "container", + binary = ":pdm", +) + +container_test( + name = "container_test", + configs = ["container-test.yaml"], + image = ":container", +) diff --git a/bazel/tools/pdm/container-test.yaml b/bazel/tools/pdm/container-test.yaml new file mode 100644 index 0000000..38a76e4 --- /dev/null +++ b/bazel/tools/pdm/container-test.yaml @@ -0,0 +1,35 @@ +schemaVersion: 2.0.0 + +metadataTest: + entrypoint: ["/bazel/tools/pdm/xpdm/pdm.runfiles/python_x86_64-unknown-linux-gnu/bin/python3"] + cmd: ["pdm"] + env: + - key: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + workdir: "/bazel/tools/pdm/xpdm" + user: 0 + +fileExistenceTests: + - name: "python3 interpreter" + path: "/bazel/tools/pdm/xpdm/pdm.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" + shouldExist: true + permissions: "-rwxr-xr-x" + +fileContentTests: + - name: "entrypoint" + path: "/bazel/tools/pdm/xpdm/pdm" + expectedContents: ["python_imports = 'pypi_deps_for_pdm/_lock/blinker@1.7.0/site-packages:pypi_deps_for_pdm/_lock/filelock@3.13.1/site-packages:pypi_deps_for_pdm/_lock/msgpack@1.0.7/site-packages:pypi_deps_for_pdm/_lock/certifi@2024.2.2/site-packages:pypi_deps_for_pdm/_lock/charset-normalizer@3.3.2/site-packages:pypi_deps_for_pdm/_lock/idna@3.6/site-packages:pypi_deps_for_pdm/_lock/urllib3@1.26.18/site-packages:pypi_deps_for_pdm/_lock/requests@2.31.0/site-packages:pypi_deps_for_pdm/_lock/cachecontrol@0.14.0/site-packages:pypi_deps_for_pdm/_lock/packaging@23.2/site-packages:pypi_deps_for_pdm/_lock/dep-logic@0.0.4/site-packages:pypi_deps_for_pdm/_lock/findpython@0.4.1/site-packages:pypi_deps_for_pdm/_lock/installer@0.7.0/site-packages:pypi_deps_for_pdm/_lock/platformdirs@4.2.0/site-packages:pypi_deps_for_pdm/_lock/tomli@2.0.1/site-packages:pypi_deps_for_pdm/_lock/pyproject-hooks@1.0.0/site-packages:pypi_deps_for_pdm/_lock/python-dotenv@1.0.1/site-packages:pypi_deps_for_pdm/_lock/requests-toolbelt@1.0.0/site-packages:pypi_deps_for_pdm/_lock/resolvelib@1.0.1/site-packages:pypi_deps_for_pdm/_lock/mdurl@0.1.2/site-packages:pypi_deps_for_pdm/_lock/markdown-it-py@3.0.0/site-packages:pypi_deps_for_pdm/_lock/pygments@2.17.2/site-packages:pypi_deps_for_pdm/_lock/rich@13.7.0/site-packages:pypi_deps_for_pdm/_lock/shellingham@1.5.4/site-packages:pypi_deps_for_pdm/_lock/tomlkit@0.12.3/site-packages:pypi_deps_for_pdm/_lock/truststore@0.8.0/site-packages:pypi_deps_for_pdm/_lock/unearth@0.14.0/site-packages:pypi_deps_for_pdm/_lock/distlib@0.3.8/site-packages:pypi_deps_for_pdm/_lock/virtualenv@20.25.0/site-packages:pypi_deps_for_pdm/_lock/pdm@2.12.3/site-packages'"] + +commandTests: + - name: "bazel python3 version" + command: "/bazel/tools/pdm/xpdm/pdm.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" + args: ["--version"] + expectedOutput: ["Python 3.10.11"] + - name: "distroless python3 version" + command: "python3" + args: ["--version"] + expectedOutput: ["Python 3.11.2"] + - name: "bazel pdm version" + command: "/bazel/tools/pdm/xpdm/pdm.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" + args: ["/bazel/tools/pdm/xpdm/pdm", "--version"] + expectedOutput: ["PDM, version 2.12.3"] diff --git a/bazel/tools/pdm/image-test.yaml b/bazel/tools/pdm/image-test.yaml new file mode 100644 index 0000000..ff5e791 --- /dev/null +++ b/bazel/tools/pdm/image-test.yaml @@ -0,0 +1,35 @@ +schemaVersion: 2.0.0 + +metadataTest: + entrypoint: ["/bazel/tools/pdm/ypdm/pdm.runfiles/python_x86_64-unknown-linux-gnu/bin/python3"] + cmd: ["pdm"] + envVars: + - key: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + workdir: "/bazel/tools/pdm/ypdm" + user: 0 + +fileExistenceTests: + - name: "python3 interpreter" + path: "/bazel/tools/pdm/ypdm/pdm.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" + shouldExist: true + permissions: "-rwxr-xr-x" + +fileContentTests: + - name: "entrypoint" + path: "/bazel/tools/pdm/ypdm/pdm" + expectedContents: ["python_imports = 'pypi_deps_for_pdm/_lock/blinker@1.7.0/site-packages:pypi_deps_for_pdm/_lock/filelock@3.13.1/site-packages:pypi_deps_for_pdm/_lock/msgpack@1.0.7/site-packages:pypi_deps_for_pdm/_lock/certifi@2024.2.2/site-packages:pypi_deps_for_pdm/_lock/charset-normalizer@3.3.2/site-packages:pypi_deps_for_pdm/_lock/idna@3.6/site-packages:pypi_deps_for_pdm/_lock/urllib3@1.26.18/site-packages:pypi_deps_for_pdm/_lock/requests@2.31.0/site-packages:pypi_deps_for_pdm/_lock/cachecontrol@0.14.0/site-packages:pypi_deps_for_pdm/_lock/packaging@23.2/site-packages:pypi_deps_for_pdm/_lock/dep-logic@0.0.4/site-packages:pypi_deps_for_pdm/_lock/findpython@0.4.1/site-packages:pypi_deps_for_pdm/_lock/installer@0.7.0/site-packages:pypi_deps_for_pdm/_lock/platformdirs@4.2.0/site-packages:pypi_deps_for_pdm/_lock/tomli@2.0.1/site-packages:pypi_deps_for_pdm/_lock/pyproject-hooks@1.0.0/site-packages:pypi_deps_for_pdm/_lock/python-dotenv@1.0.1/site-packages:pypi_deps_for_pdm/_lock/requests-toolbelt@1.0.0/site-packages:pypi_deps_for_pdm/_lock/resolvelib@1.0.1/site-packages:pypi_deps_for_pdm/_lock/mdurl@0.1.2/site-packages:pypi_deps_for_pdm/_lock/markdown-it-py@3.0.0/site-packages:pypi_deps_for_pdm/_lock/pygments@2.17.2/site-packages:pypi_deps_for_pdm/_lock/rich@13.7.0/site-packages:pypi_deps_for_pdm/_lock/shellingham@1.5.4/site-packages:pypi_deps_for_pdm/_lock/tomlkit@0.12.3/site-packages:pypi_deps_for_pdm/_lock/truststore@0.8.0/site-packages:pypi_deps_for_pdm/_lock/unearth@0.14.0/site-packages:pypi_deps_for_pdm/_lock/distlib@0.3.8/site-packages:pypi_deps_for_pdm/_lock/virtualenv@20.25.0/site-packages:pypi_deps_for_pdm/_lock/pdm@2.12.3/site-packages'"] + +commandTests: + - name: "bazel python3 version" + command: "/bazel/tools/pdm/ypdm/pdm.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" + args: ["--version"] + expectedOutput: ["Python 3.10.11"] + - name: "distroless python3 version" + command: "python3" + args: ["--version"] + expectedOutput: ["Python 3.11.2"] + - name: "bazel pdm version" + command: "/bazel/tools/pdm/ypdm/pdm.runfiles/python_x86_64-unknown-linux-gnu/bin/python3" + args: ["/bazel/tools/pdm/ypdm/pdm", "--version"] + expectedOutput: ["PDM, version 2.12.3"] diff --git a/bazel/tools/pdm/run.sh b/bazel/tools/pdm/run.sh deleted file mode 100755 index 5eee1d8..0000000 --- a/bazel/tools/pdm/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -export PDM_PROJECT=$BUILD_WORKSPACE_DIRECTORY - -./tools/pdm/pdm $@ diff --git a/bazel/workspace/repositories.bzl b/bazel/workspace/repositories.bzl index 3daf931..1e5726c 100644 --- a/bazel/workspace/repositories.bzl +++ b/bazel/workspace/repositories.bzl @@ -42,19 +42,26 @@ def repositories(): urls = ["https://github.com/bazelbuild/bazel-skylib/archive/1.5.0.tar.gz"], ) - # Commit: c010a3b95427ddfa678e92250e4bcf7d95cf39ce - # Date: 2024-02-27 16:03:55 +0000 UTC - # URL: https://github.com/GoogleContainerTools/container-structure-test/commit/c010a3b95427ddfa678e92250e4bcf7d95cf39ce - # - # Release v1.17.0 - # - # Signed-off-by: Appu Goundan - # Size: 61740 (62 kB) + # # Commit: c010a3b95427ddfa678e92250e4bcf7d95cf39ce + # # Date: 2024-02-27 16:03:55 +0000 UTC + # # URL: https://github.com/GoogleContainerTools/container-structure-test/commit/c010a3b95427ddfa678e92250e4bcf7d95cf39ce + # # + # # Release v1.17.0 + # # + # # Signed-off-by: Appu Goundan + # # Size: 61740 (62 kB) + # http_archive( + # name = "container_structure_test", + # sha256 = "df5043d1edef7c06f8ba94cb8b383b4a9cb4bdcf90db4b9c312a6a8ddd336916", + # strip_prefix = "container-structure-test-c010a3b95427ddfa678e92250e4bcf7d95cf39ce", + # urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/c010a3b95427ddfa678e92250e4bcf7d95cf39ce.tar.gz"], + # ) + http_archive( name = "container_structure_test", - sha256 = "df5043d1edef7c06f8ba94cb8b383b4a9cb4bdcf90db4b9c312a6a8ddd336916", - strip_prefix = "container-structure-test-c010a3b95427ddfa678e92250e4bcf7d95cf39ce", - urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/c010a3b95427ddfa678e92250e4bcf7d95cf39ce.tar.gz"], + sha256 = "6e5e3cbb15f2785c652822a56e0c86b14b630a6a3a410c889e718d580f2c19a7", + strip_prefix = "container-structure-test-18d9b2bcd2cbee1ab8df7c232ef6b7cddff9708a", + urls = ["https://github.com/GoogleContainerTools/container-structure-test/archive/18d9b2bcd2cbee1ab8df7c232ef6b7cddff9708a.zip"], ) # Release: v0.25.0 diff --git a/bazel/workspace/step1.bzl b/bazel/workspace/step1.bzl index beb37f9..f2a9487 100644 --- a/bazel/workspace/step1.bzl +++ b/bazel/workspace/step1.bzl @@ -42,6 +42,7 @@ load( "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains", ) +load("@container_structure_test//:repositories.bzl", "container_structure_test_register_toolchain") def _setup_zig_toolchains(): "workspace chunk to declares and registers zig toolchains" @@ -86,6 +87,11 @@ def _setup_rules_oci(): crane_version = LATEST_CRANE_VERSION, ) +def _setup_container_structure_test(): + container_structure_test_register_toolchain( + name = "container_structure_test_toolchain", + ) + step1 = struct( setup_aspect_bazel_lib = _setup_aspect_bazel_lib, setup_bazel_gazelle = _setup_bazel_gazelle, @@ -95,4 +101,5 @@ step1 = struct( setup_rules_oci = _setup_rules_oci, setup_rules_python = _setup_rules_python, setup_zig_toolchains = _setup_zig_toolchains, + setup_container_structure_test = _setup_container_structure_test, ) diff --git a/bazel/workspace/step2.bzl b/bazel/workspace/step2.bzl index 813c305..e84a8e9 100644 --- a/bazel/workspace/step2.bzl +++ b/bazel/workspace/step2.bzl @@ -19,6 +19,13 @@ load( "oci_pull", ) +# from 'crane manifest gcr.io/distroless/python3-debian12:latest' circa Jan 2024 +python3_debian12_image = struct( + registry = "gcr.io", + repository = "distroless/python3-debian12", + digest = "sha256:0078c63ba4e9bb13eef1576a183fc0bc3fd04fd3d5a9bad5ede1069bddca0ebd", +) + def _setup_rules_pycross( pycross_toolchains_repo_name = "pycross_toolchains", glibc_version = "2.28"): @@ -31,35 +38,33 @@ def _setup_rules_pycross( python_toolchains_repo = "@python", ) -def _setup_oci_containers(): - oci_pull( - name = "distroless_base", - digest = "sha256:ccaef5ee2f1850270d453fdf700a5392534f8d1a8ca2acda391fbb6a06b81c86", - image = "gcr.io/distroless/base", - platforms = [ - "linux/amd64", - "linux/arm64", - ], - ) +def _setup_pycross_image_base_oci(**kwargs): + digest = kwargs.pop("digest", python3_debian12_image.digest) + image = kwargs.pop("image", "%s/%s" % (python3_debian12_image.registry, python3_debian12_image.repository)) oci_pull( - name = "distroless_python3_debian12_oci", - digest = "sha256:0078c63ba4e9bb13eef1576a183fc0bc3fd04fd3d5a9bad5ede1069bddca0ebd", - image = "gcr.io/distroless/python3-debian12", + name = "pycross_image_base_oci", + digest = digest, + image = image, + **kwargs ) -def _setup_docker_containers(): +def _setup_pycross_image_base_container(**kwargs): + digest = kwargs.pop("digest", python3_debian12_image.digest) + registry = kwargs.pop("image", python3_debian12_image.registry) + repository = kwargs.pop("repository", python3_debian12_image.repository) + container_pull( - name = "distroless_python3_debian12_container", - # from 'crane manifest gcr.io/distroless/python3-debian12:latest' - digest = "sha256:0078c63ba4e9bb13eef1576a183fc0bc3fd04fd3d5a9bad5ede1069bddca0ebd", - registry = "gcr.io", - repository = "distroless/python3-debian12", - # docker_client_config = "@pycross_image//:.dockerconfig.json", + name = "pycross_image_base_container", + digest = digest, + # tag = "debug", # enable for debugging e.g 'docker run -it --entrypoint=sh bazel/bazel/tools/pdm:container' + registry = registry, + repository = repository, + **kwargs ) step2 = struct( setup_rules_pycross = _setup_rules_pycross, - setup_docker_containers = _setup_docker_containers, - setup_oci_containers = _setup_oci_containers, + setup_pycross_image_base_container = _setup_pycross_image_base_container, + setup_pycross_image_base_oci = _setup_pycross_image_base_oci, ) diff --git a/example/BUILD.bazel b/example/BUILD.bazel index 9d246b6..bf97c55 100644 --- a/example/BUILD.bazel +++ b/example/BUILD.bazel @@ -21,7 +21,6 @@ example_test( srcs = [ "docker/.bazelrc", "docker/.bazelversion", - "docker/.docker/config.json", "docker/.dockerconfig.json", "docker/BUILD.in", "docker/WORKSPACE.in", diff --git a/example/docker/.bazelrc b/example/docker/.bazelrc index 5273e7d..e69de29 100644 --- a/example/docker/.bazelrc +++ b/example/docker/.bazelrc @@ -1,2 +0,0 @@ ---repo_env=DOCKER_CONFIG=. ---spawn_strategy=standalone \ No newline at end of file diff --git a/example/docker/.docker/config.json b/example/docker/.docker/config.json deleted file mode 100644 index 9e26dfe..0000000 --- a/example/docker/.docker/config.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/example/docker/BUILD.in b/example/docker/BUILD.in index b9a8971..e69de29 100644 --- a/example/docker/BUILD.in +++ b/example/docker/BUILD.in @@ -1,5 +0,0 @@ -exports_files([ - "pyproject.toml", - "pdm.lock", - ".dockerconfig.json", -]) diff --git a/example/docker/WORKSPACE.in b/example/docker/WORKSPACE.in index 5db4c3c..c637bea 100644 --- a/example/docker/WORKSPACE.in +++ b/example/docker/WORKSPACE.in @@ -26,12 +26,11 @@ load("@pycross_image//bazel/workspace:step2.bzl", "step2") step2.setup_rules_pycross() - # ----------------------------------------- load("@//:step3.bzl", "step3") -step3.setup_docker_containers() +step3.setup_container_base_image() step3.setup_pypi_deps() diff --git a/example/docker/src/app/BUILD.in b/example/docker/src/app/BUILD.in index 04b76f0..ca6ee8f 100644 --- a/example/docker/src/app/BUILD.in +++ b/example/docker/src/app/BUILD.in @@ -12,5 +12,6 @@ py_binary( py_image( name = "image", + base = "@distroless_python3_debian12_container//image", binary = ":app", ) diff --git a/example/docker/step3.bzl b/example/docker/step3.bzl index c47a835..33f513a 100644 --- a/example/docker/step3.bzl +++ b/example/docker/step3.bzl @@ -11,14 +11,18 @@ load( "@io_bazel_rules_docker//container:pull.bzl", "container_pull", ) +load("@pycross_image//bazel/workspace:step2.bzl", "python3_debian12_image") + +def _setup_container_base_image(**kwargs): + digest = kwargs.pop("digest", python3_debian12_image.digest) + registry = kwargs.pop("image", python3_debian12_image.registry) + repository = kwargs.pop("repository", python3_debian12_image.repository) -def _setup_docker_containers(): container_pull( name = "distroless_python3_debian12_container", - # from 'crane manifest gcr.io/distroless/python3-debian12:latest' - digest = "sha256:0078c63ba4e9bb13eef1576a183fc0bc3fd04fd3d5a9bad5ede1069bddca0ebd", - registry = "gcr.io", - repository = "distroless/python3-debian12", + digest = digest, + registry = registry, + repository = repository, docker_client_config = "//:.dockerconfig.json", ) @@ -39,6 +43,6 @@ def _setup_pypi_deps(): ) step3 = struct( - setup_docker_containers = _setup_docker_containers, + setup_container_base_image = _setup_container_base_image, setup_pypi_deps = _setup_pypi_deps, )