How to build librdkafka #1995

UebelAndre · 2023-06-07T11:34:47Z

UebelAndre
Jun 7, 2023
Maintainer

I saw this incredible thread in the slack channel and thought I'd re-post it here to make it more discoverable

David Lopez:

has someone been able to build the the rdkafka library using the cmake-build feature using bazel? Im getting panics after the following warnings:
Building and linking librdkafka statically
CMake Error: Could not find CMAKE_ROOT !!!
CMake has most likely not been installed correctly.
Modules directory not found in
I defined CMAKE_ROOT and CMAKE_MODULE_PATH in my shell but seems like bazel is ignoring it and it is using a different cmake version that the one that I have installed in my system (edited)

This then lead to a hero's reponse

Brian Myers:

Here's what my librdkafka looks like to help get you started. The others (zstd, lz4, zlib, openssl) should have examples if you google for them. I don't recall seeing any good librdkafka examples, so pasting here

load("@bazel_skylib//rules:write_file.bzl", "write_file")
load("@rules_cc//cc:defs.bzl", "cc_library")

alias(
    name = "librdkafka",
    actual = "rdkafka",
    visibility = ["//visibility:public"],
)

alias(
    name = "librdkafka++",
    actual = "rdkafka++",
    visibility = ["//visibility:public"],
)

write_file(
    name = "config_dot_h",
    out = "config.h",
    content = [
        "#define WITHOUT_OPTIMIZATION 0",
        "#define ENABLE_DEVEL 0",
        "#define ENABLE_REFCNT_DEBUG 0",
        "#define HAVE_ATOMICS_32 1",
        "#define HAVE_ATOMICS_32_SYNC 0",
        "#if (HAVE_ATOMICS_32)",
        "# if (HAVE_ATOMICS_32_SYNC)",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_ATOMICS_64 1",
        "#define HAVE_ATOMICS_64_SYNC 0",
        "#if (HAVE_ATOMICS_64)",
        "# if (HAVE_ATOMICS_64_SYNC)",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_RAND_R 1",
        "#define HAVE_REGEX 1",
        "#define HAVE_STRNDUP 1",
        "#define WITH_C11THREADS 0",
        "#define WITH_CRC32C_HW 0",
        "#define WITH_HDRHISTOGRAM 1",
        "#define WITH_LIBDL 1",
        "#define WITH_LZ4_EXT 1",
        "#define WITH_PKGCONFIG 1",
        "#define WITH_PLUGINS 1",
        "#define WITH_SASL 0",
        "#define WITH_SASL_CYRUS 0",
        "#define WITH_SASL_OAUTHBEARER 0",
        "#define WITH_SASL_SCRAM 0",
        "#define WITH_SNAPPY 1",
        "#define WITH_SOCKEM 1",
        "#define WITH_SSL 1",
        "#define WITH_ZLIB 1",
        "#define WITH_ZSTD 1",
    ] + select({
        "@//bazel/constraints/os:darwin": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 1",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 0",
            "#define SOLIB_EXT \".dylib\"",
            # The compiler here is incorrect, but I don't think it matters
            "#define BUILT_WITH  \"CMAKE AppleClang AppleClang PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "@//bazel/constraints/os:linux": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 0",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 1",
            "#define SOLIB_EXT \".so\"",
            "#define BUILT_WITH  \"CMAKE GNU GNU PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "//conditions:default": [],
    }),
)

cc_library(
    name = "config_header",
    hdrs = [
        "config.h",
    ],
)

cc_library(
    name = "rdkafka_header",
    hdrs = [
        "src/rdkafka.h",
    ],
)

cc_library(
    name = "rdkafka",
    srcs = glob(
        include = [
            "src/*.c",
            "src/*.h",
        ],
        exclude = [
            # don't use sasl
            "src/rdkafka_sasl_cyrus.c",
            "src/rdkafka_sasl_oauthbearer.c",
            "src/rdkafka_sasl_scram.c",
            "src/rdkafka_sasl_win32.c",
            # no need for win32
            "src/rdwin32.h",
            "src/win32_config.h",
            # don't use built-in laz4
            "src/lz4.c",
            "src/lz4frame.c",
            "src/lz4hc.c",
        ],
    ),
    hdrs = [
        "src/rdkafka.h",
    ],
    copts = [
        "-fPIC",
        "-Wall",
        # Keep log spew down when compiling with LLVM
        "-Wno-unused-but-set-variable",
    ],
    include_prefix = "librdkafka",
    strip_include_prefix = "src",
    deps = [
        ":config_header",
        "@com_github__facebook__zstd//:libzstd",
        "@com_github__lz4__lz4//:liblz4",
        "@com_github__openssl__openssl//:openssl",
        "@net_zlib__zlib//:libz",
    ],
)

cc_library(
    name = "rdkafka++",
    srcs = glob([
        "src-cpp/*.cpp",
        "src-cpp/*.h",
    ]),
    hdrs = [
        "src-cpp/rdkafkacpp.h",
    ],
    strip_include_prefix = "src-cpp",
    deps = [
        ":config_header",
        ":rdkafka_header",
    ],
)

I only have to support macos and linux by the way. Good luck if you care about windows
Oh, and make sure you pull the version of librdkafka that rdkafka-sys uses. rdkafka 0.28.0 uses https://github.com/benesch/librdkafka/archive/30633e59a9072b7b9c76411c281578e0125d5f65.tar.gz

Huge thanks to Brian for sharing!

UebelAndre · 2025-03-10T20:25:24Z

UebelAndre
Mar 10, 2025
Maintainer Author

There's now https://github.com/scasagrande/bazel-examples/tree/main/rdkafka-sys !

"""
This file contains a native bazel build for librdkafka.

This implementation is a modified version of this one that was posted
in the bazel slack:
https://bazelbuild.slack.com/archives/CSV56UT0F/p1685744596942669?thread_ts=1685743903.751189&cid=CSV56UT0F

While messy, there is nothing actually special about it. The standard
librdkafka build systems (cmake and mklove) will construct this `config.h`
file that we have just manually defined here in this BUILD.bazel file.
"""

load("@bazel_skylib//rules:write_file.bzl", "write_file")
load("@rules_cc//cc:defs.bzl", "cc_library")

alias(
    name = "librdkafka",
    actual = "rdkafka",
    visibility = ["//visibility:public"],
)

alias(
    name = "librdkafka++",
    actual = "rdkafka++",
    visibility = ["//visibility:public"],
)

write_file(
    name = "config_dot_h",
    out = "config.h",
    content = [
        "#define WITHOUT_OPTIMIZATION 0",
        "#define ENABLE_DEVEL 0",
        "#define ENABLE_REFCNT_DEBUG 0",
        "#define HAVE_ATOMICS_32 1",
        "#define HAVE_ATOMICS_32_SYNC 0",
        "#if (HAVE_ATOMICS_32)",
        "# if (HAVE_ATOMICS_32_SYNC)",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_ATOMICS_64 1",
        "#define HAVE_ATOMICS_64_SYNC 0",
        "#if (HAVE_ATOMICS_64)",
        "# if (HAVE_ATOMICS_64_SYNC)",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_RAND_R 1",
        "#define HAVE_REGEX 1",
        "#define HAVE_STRNDUP 1",
        "#define WITH_C11THREADS 0",
        "#define WITH_CRC32C_HW 0",
        "#define WITH_HDRHISTOGRAM 1",
        "#define WITH_LIBDL 1",
        "#define WITH_LZ4_EXT 1",
        "#define WITH_PKGCONFIG 1",
        "#define WITH_PLUGINS 1",
        "#define WITH_SASL 0",  # flip for sasl support
        "#define WITH_SASL_CYRUS 0",  # flip for sasl support
        "#define WITH_SASL_OAUTHBEARER 0",
        "#define WITH_SASL_SCRAM 0",
        "#define WITH_SNAPPY 1",
        "#define WITH_SOCKEM 1",
        "#define WITH_SSL 1",
        "#define WITH_ZLIB 0",
        "#define WITH_ZSTD 0",
    ] + select({
        "@platforms//os:macos": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 1",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 0",
            "#define SOLIB_EXT \".dylib\"",
            # The compiler here is incorrect, but I don't think it matters
            "#define BUILT_WITH  \"CMAKE AppleClang AppleClang PKGCONFIG HDRHISTOGRAM LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
            #            "#define BUILT_WITH  \"CMAKE AppleClang AppleClang PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "@platforms//os:linux": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 0",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 1",
            "#define SOLIB_EXT \".so\"",
            "#define BUILT_WITH  \"CMAKE GNU GNU PKGCONFIG HDRHISTOGRAM LIBDL PLUGINS SSL SASL_CYRUS SASL_SCRAM LZ4_EXT SNAPPY SOCKEM\"",
            #            "#define BUILT_WITH  \"CMAKE GNU GNU PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "//conditions:default": [],
    }),
)

cc_library(
    name = "config_header",
    hdrs = [
        "config.h",
    ],
)

cc_library(
    name = "rdkafka_header",
    hdrs = [
        "src/rdkafka.h",
    ],
)

cc_library(
    name = "rdkafka",
    srcs = glob(
        include = [
            "src/*.c",
            "src/*.h",
        ],
        exclude = [
            # Comment out to include sasl support
            "src/rdkafka_sasl_cyrus.c",
            "src/rdkafka_sasl_scram.c",

            # don't use sasl oauthbearer, it needs libcurl-dev
            "src/rdkafka_sasl_oauthbearer.c",
            "src/rdkafka_sasl_oauthbearer_oidc.c",
            "src/rdhttp.c",

            # no need for win32
            "src/rdkafka_sasl_win32.c",
            "src/rdwin32.h",
            "src/win32_config.h",

            # don't use built-in lz4
            "src/lz4.c",
            "src/lz4frame.c",
            "src/lz4hc.c",

            # don't use zstd
            "src/rdkafka_zstd.c",

            # don't use zlib
            "src/rdgz.c",

            # These are also defined in @lz4//:lib/xxhash.c
            "src/rdxxhash.c",
        ],
    ),
    hdrs = [
        "src/rdkafka.h",
    ],
    #    copts = [
    #        "-fPIC",
    #        "-Wall",
    #        # Keep log spew down when compiling with LLVM
    #        "-Wno-unused-but-set-variable",
    #    ],
    data = [":config.h"],
    include_prefix = "librdkafka",
    strip_include_prefix = "src",
    deps = [
        ":config_header",
        "@lz4",
        "@lz4//:lz4_frame",
        "@openssl",
    ],
)

cc_library(
    name = "rdkafka++",
    srcs = glob([
        "src-cpp/*.cpp",
        "src-cpp/*.h",
    ]),
    hdrs = [
        "src-cpp/rdkafkacpp.h",
    ],
    strip_include_prefix = "src-cpp",
    deps = [
        ":config_header",
        ":rdkafka_header",
    ],
)

0 replies

UebelAndre · 2025-03-10T20:26:27Z

UebelAndre
Mar 10, 2025
Maintainer Author

And for bzlmod, seems like there's https://github.com/bazelbuild/bazel-central-registry/tree/eb0fdf46f22a8df0a8a66e61716a27068888ea55/modules/librdkafka

0 replies

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

How to build librdkafka #1995

Uh oh!

{{title}}

Uh oh!

Replies: 2 comments

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Select a reply

Uh oh!

How to build librdkafka #1995

Uh oh!

UebelAndre Jun 7, 2023 Maintainer

Replies: 2 comments

Uh oh!

UebelAndre Mar 10, 2025 Maintainer Author

Uh oh!

UebelAndre Mar 10, 2025 Maintainer Author

UebelAndre
Jun 7, 2023
Maintainer

UebelAndre
Mar 10, 2025
Maintainer Author

UebelAndre
Mar 10, 2025
Maintainer Author