From 97558d3d59aecb6bc71c82a18953c3537233c934 Mon Sep 17 00:00:00 2001 From: zapdos26 Date: Tue, 20 Aug 2024 16:48:06 -0400 Subject: [PATCH] enhancement(kafka sink): Allow OIDC usage for Kafka (#21103) * Adding "cmake" and "curl" required for OIDC to work with rdkafka * Revert unneccessary update to rdkafka * Removed cmake-build * Create 21103_kafka_sink_oidc_authentication.enhancement.md * Add oidc to expected spelling * Update license file * Update Cargo.toml --------- Co-authored-by: Jesse Szwedko --- .github/actions/spelling/expect.txt | 1 + Cargo.lock | 18 +++++++++++++++++- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 + ...fka_sink_oidc_authentication.enhancement.md | 3 +++ 5 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 changelog.d/21103_kafka_sink_oidc_authentication.enhancement.md diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index ddd4816bfe92e..bdc3e9b5b79c4 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -779,6 +779,7 @@ nullishness numbackends oahd oap +oidc OKD omfwd omitempty diff --git a/Cargo.lock b/Cargo.lock index 90f545917a7c4..961b3598a9917 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2607,6 +2607,21 @@ dependencies = [ "cipher", ] +[[package]] +name = "curl-sys" +version = "0.4.74+curl-8.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8af10b986114528fcdc4b63b6f5f021b7057618411046a4de2ba0f0149a097bf" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "windows-sys 0.52.0", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -4340,7 +4355,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.7", + "socket2 0.4.10", "tokio", "tower-service", "tracing 0.1.40", @@ -7548,6 +7563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" dependencies = [ "cmake", + "curl-sys", "libc", "libz-sys", "num_enum 0.5.11", diff --git a/Cargo.toml b/Cargo.toml index 3cc6c94864a9d..3f43e9a18b5e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -324,7 +324,7 @@ postgres-openssl = { version = "0.5.0", default-features = false, features = ["r pulsar = { version = "6.3.0", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } rand = { version = "0.8.5", default-features = false, features = ["small_rng"] } rand_distr = { version = "0.4.3", default-features = false } -rdkafka = { version = "0.35.0", default-features = false, features = ["tokio", "libz", "ssl", "zstd"], optional = true } +rdkafka = { version = "0.35.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } redis = { version = "0.24.0", default-features = false, features = ["connection-manager", "tokio-comp", "tokio-native-tls-comp"], optional = true } regex = { version = "1.10.6", default-features = false, features = ["std", "perf"] } roaring = { version = "0.10.6", default-features = false, features = ["std"], optional = true } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ea585e64fa881..b4e8cabb5b579 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -160,6 +160,7 @@ crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers +curl-sys,https://github.com/alexcrichton/curl-rust,MIT,Alex Crichton curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/curve25519-dalek,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs diff --git a/changelog.d/21103_kafka_sink_oidc_authentication.enhancement.md b/changelog.d/21103_kafka_sink_oidc_authentication.enhancement.md new file mode 100644 index 0000000000000..3b834393df8bf --- /dev/null +++ b/changelog.d/21103_kafka_sink_oidc_authentication.enhancement.md @@ -0,0 +1,3 @@ +The `kafka` sink now supports OIDC authentication through the exposed `librdkafka_options`. + +authors: zapdos26