feat(kafka): add zeromq kafka bridge

hongbo-miao · Nov 15, 2024 · e27e615 · e27e615
1 parent 4bd85fc
commit e27e615
Show file tree

Hide file tree

Showing 9 changed files with 2,512 additions and 0 deletions.
diff --git a/.github/workflows/.lint.yml b/.github/workflows/.lint.yml
@@ -24,6 +24,7 @@ jobs:
       kafka-rust-proto-consumer: ${{ steps.filter.outputs.kafka-rust-proto-consumer }}
       kafka-rust-proto-producer: ${{ steps.filter.outputs.kafka-rust-proto-producer }}
       kafka-rust-udp-kafka-bridge: ${{ steps.filter.outputs.kafka-rust-udp-kafka-bridge }}
+      kafka-rust-zeromq-kafka-bridge: ${{ steps.filter.outputs.kafka-rust-zeromq-kafka-bridge }}
       kubernetes: ${{ steps.filter.outputs.kubernetes }}
       udp-receiver: ${{ steps.filter.outputs.udp-receiver }}
       udp-sender: ${{ steps.filter.outputs.udp-sender }}
@@ -63,6 +64,9 @@ jobs:
             kafka-rust-udp-kafka-bridge:
               - '.github/workflows/.lint.yml'
               - 'hm-kafka/kafka-client/kafka-rust/udp-kafka-bridge/**'
+            kafka-rust-zeromq-kafka-bridge:
+              - '.github/workflows/.lint.yml'
+              - 'hm-kafka/kafka-client/kafka-rust/zeromq-kafka-bridge/**'
             kubernetes:
               - '.github/workflows/.lint.yml'
               - 'kubernetes/manifests/**'
@@ -662,6 +666,44 @@ jobs:
           cargo clippy
           cargo fmt --all
 
+  lint-rust-kafka-rust-zeromq-kafka-bridge:
+    name: Rust (kafka-rust-zeromq-kafka-bridge)
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.kafka-rust-zeromq-kafka-bridge == 'true' }}
+    runs-on: ubuntu-22.04
+    environment: test
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      # librdkafka is for rdkafka
+      - name: Install librdkafka
+        run: |
+          # https://github.com/edenhill/librdkafka#build-from-source
+          wget --no-verbose --output-document=librdkafka.tar.gz https://github.com/edenhill/librdkafka/archive/refs/tags/v2.6.0.tar.gz
+          tar -x -f librdkafka.tar.gz
+          rm -f librdkafka.tar.gz
+          cd librdkafka-2.6.0
+          ./configure
+          make
+          sudo make install
+          sudo ldconfig
+          cd ..
+          rm -r -f librdkafka-2.6.0/
+      # protoc is for prost
+      - name: Install protoc
+        uses: arduino/[email protected]
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          version: 28.3
+      - name: Set up Rust
+        uses: actions-rust-lang/[email protected]
+      - name: Lint Rust
+        working-directory: hm-kafka/kafka-client/kafka-rust/zeromq-kafka-bridge
+        run: |
+          cargo clippy
+          cargo fmt --all
+
   lint-rust-udp-receiver:
     name: Rust (udp-receiver)
     needs: detect-changes

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -88,6 +88,7 @@ jobs:
       kafka-rust-proto-consumer: ${{ steps.filter.outputs.kafka-rust-proto-consumer }}
       kafka-rust-proto-producer: ${{ steps.filter.outputs.kafka-rust-proto-producer }}
       kafka-rust-udp-kafka-bridge: ${{ steps.filter.outputs.kafka-rust-udp-kafka-bridge }}
+      kafka-rust-zeromq-kafka-bridge: ${{ steps.filter.outputs.kafka-rust-zeromq-kafka-bridge }}
       mobile-android: ${{ steps.filter.outputs.mobile-android }}
       mobile-ios: ${{ steps.filter.outputs.mobile-ios }}
       mobile-react-native: ${{ steps.filter.outputs.mobile-react-native }}
@@ -192,6 +193,9 @@ jobs:
             kafka-rust-udp-kafka-bridge:
               - '.github/workflows/test.yml'
               - 'hm-kafka/kafka-client/kafka-rust/udp-kafka-bridge/**'
+            kafka-rust-zeromq-kafka-bridge:
+              - '.github/workflows/test.yml'
+              - 'hm-kafka/kafka-client/kafka-rust/zeromq-kafka-bridge/**'
             national-instruments-hm-ni-veristand:
               - '.github/workflows/test.yml'
               - 'hardware-in-the-loop/national-instruments/hm-ni-veristand/**'
@@ -2700,6 +2704,43 @@ jobs:
         run: |
           cargo test --all-features
 
+  rust-kafka-rust-zeromq-kafka-bridge-test:
+    name: Kafka Rust - ZeroMQ Kafka Bridge | Test
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.kafka-rust-zeromq-kafka-bridge == 'true' }}
+    runs-on: ubuntu-22.04
+    environment: test
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      # librdkafka is for rdkafka
+      - name: Install librdkafka
+        run: |
+          # https://github.com/edenhill/librdkafka#build-from-source
+          wget --no-verbose --output-document=librdkafka.tar.gz https://github.com/edenhill/librdkafka/archive/refs/tags/v2.6.0.tar.gz
+          tar -x -f librdkafka.tar.gz
+          rm -f librdkafka.tar.gz
+          cd librdkafka-2.6.0
+          ./configure
+          make
+          sudo make install
+          sudo ldconfig
+          cd ..
+          rm -r -f librdkafka-2.6.0/
+      # protoc is for prost
+      - name: Install protoc
+        uses: arduino/[email protected]
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          version: 28.3
+      - name: Set up Rust
+        uses: actions-rust-lang/[email protected]
+      - name: Test
+        working-directory: hm-kafka/kafka-client/kafka-rust/zeromq-kafka-bridge
+        run: |
+          cargo test --all-features
+
   udp-receiver-test:
     name: UDP Receiver | Test
     needs: detect-changes

diff --git a/.mergify.yml b/.mergify.yml
@@ -77,6 +77,12 @@ pull_request_rules:
       - or:
           - check-success=Lint / Rust (kafka-rust-proto-producer)
           - check-skipped=Lint / Rust (kafka-rust-proto-producer)
+      - or:
+          - check-success=Lint / Rust (kafka-rust-udp-kafka-bridge)
+          - check-skipped=Lint / Rust (kafka-rust-udp-kafka-bridge)
+      - or:
+          - check-success=Lint / Rust (kafka-rust-zeromq-kafka-bridge)
+          - check-skipped=Lint / Rust (kafka-rust-zeromq-kafka-bridge)
       - or:
           - check-success=Lint / Rust (udp-receiver)
           - check-skipped=Lint / Rust (udp-receiver)
@@ -362,6 +368,9 @@ pull_request_rules:
       - or:
           - check-success=Kafka Rust - UDP Kafka Bridge | Test
           - check-skipped=Kafka Rust - UDP Kafka Bridge | Test
+      - or:
+          - check-success=Kafka Rust - ZeroMQ Kafka Bridge | Test
+          - check-skipped=Kafka Rust - ZeroMQ Kafka Bridge | Test
       - or:
           - check-success=UDP Receiver | Test
           - check-skipped=UDP Receiver | Test