Initial setup and configuration for Docker Fluent Bit Collector

.github/workflows/ci-tests.yml

@@ -0,0 +1,106 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Set up Python environment
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install Python Dependencies
+        run: |
+          pip install -r requirements.txt
+
+      # Run Python unit tests
+      - name: Run Python Unit Tests
+        run: |
+          python -m unittest discover -s tests -p 'test_create_fluent_bit_config.py' -v
+
+      # Set up Lua environment
+      - name: Install Lua and LuaRocks
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y lua5.3 lua5.3-dev luarocks
+
+      - name: Install Lua Dependencies
+        run: |
+          sudo luarocks install busted
+
+      # Run Lua unit tests
+      - name: Run Lua Unit Tests
+        working-directory: tests
+        run: |
+          busted test_docker_metadata.lua
+
+  e2e-tests:
+    name: End-to-End Tests
+    needs: unit-tests
+    runs-on: ubuntu-latest
+    services:
+      docker:
+        image: docker:20.10-dind
+        options: --privileged
+    env:
+      LOGZIO_LOGS_TOKEN: ${{ secrets.LOGZIO_LOGS_TOKEN }}
+      LOGZIO_API_TOKEN: ${{ secrets.LOGZIO_API_TOKEN }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Set up Python environment
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install Python Dependencies
+        run: |
+          pip install -r requirements.txt
+
+      # Install Docker Compose
+      - name: Install Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+
+      # Pull Docker images
+      - name: Pull Docker Images
+        run: |
+          docker-compose pull
+
+      # Run Docker Compose
+      - name: Run Docker Compose
+        run: docker-compose up -d
+
+      # Wait for logs to be ingested
+      - name: Wait for Logs to be Ingested
+        run: sleep 60  # Adjust as necessary
+
+      # Run End-to-End Tests
+      - name: Run E2E Tests
+        run: python tests/test_e2e.py
+
+      # Output Docker Collector Logs
+      - name: Output Docker Collector Logs
+        if: always()
+        run: docker logs docker-logs-collector || true
+
+      # Tear down Docker Compose
+      - name: Tear Down Docker Compose
+        if: always()
+        run: docker-compose down

.github/workflows/release.yml

@@ -0,0 +1,44 @@
+name: Release
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-and-push-images:
+    name: Build and Push Docker Images
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture: [amd64, arm64]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: Dockerfile.${{ matrix.architecture }}
+          platforms: linux/${{ matrix.architecture }}
+          push: true
+          tags: |
+            logzio/docker-logs-collector-${{ matrix.architecture }}:latest
+            logzio/docker-logs-collector-${{ matrix.architecture }}:${{ github.ref_name }}
+
+      - name: Logout from Docker Hub
+        if: always()
+        run: docker logout

Dockerfile.amd64

@@ -0,0 +1,51 @@
+# Dockerfile.amd64
+
+FROM python:3.12.4-slim AS base
+
+# Install dependencies using apt-get
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    bash \
+    libyaml-dev \
+    libsystemd-dev \
+    libsasl2-dev \
+    libpq-dev \
+    openssl \
+    libssl-dev \
+    gdb \
+    && rm -rf /var/lib/apt/lists/*
+
+# Manually download and install libssl1.1
+RUN wget http://ftp.us.debian.org/debian/pool/main/o/openssl/libssl-dev_3.0.14-1~deb12u2_amd64.deb \
+    && dpkg -i libssl-dev_3.0.14-1~deb12u2_amd64.deb \
+    && rm libssl-dev_3.0.14-1~deb12u2_amd64.deb
+
+
+# Create the plugins directory and download the Logz.io plugin
+RUN mkdir -p /fluent-bit/plugins && \
+    wget -O /fluent-bit/plugins/out_logzio.so \
+    https://github.com/logzio/fluent-bit-logzio-output/raw/master/build/out_logzio-linux.so
+
+# Set working directory
+WORKDIR /opt/fluent-bit
+
+# Copy configuration files and Lua script
+COPY configs/parser_multiline.conf /fluent-bit/etc/parsers_multiline.conf
+COPY configs/parsers.conf /fluent-bit/etc/parsers.conf
+COPY configs/plugins.conf /fluent-bit/etc/plugins.conf
+COPY docker-metadata.lua /fluent-bit/etc/docker-metadata.lua
+COPY create_fluent_bit_config.py /opt/fluent-bit/docker-collector-logs/create_fluent_bit_config.py
+
+# Use official Fluent Bit image for Fluent Bit binaries
+FROM fluent/fluent-bit:3.1.4 AS fluent-bit
+
+# Copy Fluent Bit binary and plugins.conf to the base image
+FROM base
+COPY --from=fluent-bit /fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit
+
+# Copy entrypoint script
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Set the entrypoint to run the shell script
+ENTRYPOINT ["/start.sh"]

Dockerfile.arm64

@@ -0,0 +1,51 @@
+# Dockerfile.arm64
+
+FROM python:3.12.4-slim AS base
+
+# Install dependencies using apt-get
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    bash \
+    libyaml-dev \
+    libsystemd-dev \
+    libsasl2-dev \
+    libpq-dev \
+    openssl \
+    libssl-dev \
+    gdb \
+    && rm -rf /var/lib/apt/lists/*
+
+# Manually download and install libssl1.1
+RUN wget http://ftp.us.debian.org/debian/pool/main/o/openssl/libssl-dev_3.0.14-1~deb12u2_arm64.deb \
+    && dpkg -i libssl-dev_3.0.14-1~deb12u2_arm64.deb \
+    && rm libssl-dev_3.0.14-1~deb12u2_arm64.deb
+
+
+# Create the plugins directory and download the Logz.io plugin
+RUN mkdir -p /fluent-bit/plugins && \
+    wget -O /fluent-bit/plugins/out_logzio.so \
+    https://github.com/logzio/fluent-bit-logzio-output/raw/master/build/out_logzio-linux-arm64.so
+
+# Set working directory
+WORKDIR /opt/fluent-bit
+
+# Copy configuration files and Lua script
+COPY configs/parser_multiline.conf /fluent-bit/etc/parsers_multiline.conf
+COPY configs/parsers.conf /fluent-bit/etc/parsers.conf
+COPY configs/plugins.conf /fluent-bit/etc/plugins.conf
+COPY docker-metadata.lua /fluent-bit/etc/docker-metadata.lua
+COPY create_fluent_bit_config.py /opt/fluent-bit/docker-collector-logs/create_fluent_bit_config.py
+
+# Use official Fluent Bit image for Fluent Bit binaries
+FROM fluent/fluent-bit:3.1.4 AS fluent-bit
+
+# Copy Fluent Bit binary and plugins.conf to the base image
+FROM base
+COPY --from=fluent-bit /fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit
+
+# Copy entrypoint script
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Set the entrypoint to run the shell script
+ENTRYPOINT ["/start.sh"]

README.md

@@ -0,0 +1,75 @@
+# docker-logs-collector
+
+docker-logs-collector is a Docker container that uses Fluent Bit to collect logs from other Docker containers and forward those logs to your Logz.io account.
+
+To use this container, you'll set environment variables in your `docker run` command.
+docker-logs-collector uses those environment variables to generate a valid Fluent Bit configuration for the container.
+docker-logs-collector mounts docker.sock and the Docker logs directory to the container itself, allowing Fluent Bit to collect the logs and metadata.
+
+docker-logs-collector ships logs only.
+If you want to ship metrics to Logz.io, see [docker-collector-metrics](https://github.com/logzio/docker-collector-metrics).
+
+**Note:**
+- Ensure your Fluent Bit configuration matches your logging requirements and environment variables are set correctly.
+
+## docker-logs-collector setup
+
+### 1. Pull the Docker image
+
+Download the appropriate Docker image for your architecture (amd64 or arm64):
+
+```shell
+# For amd64 architecture
+docker pull logzio/docker-logs-collector-amd64:latest
+
+# For arm64 architecture
+docker pull logzio/docker-logs-collector-arm64:latest
+```
+
+### 2. Run the container
+
+Replace `<ARCH>` in the image name with either amd64 or arm64 based on your system architecture.
+
+For a complete list of options, see the parameters below the code block.👇
+
+```shell
+docker run --name docker-logs-collector \
+--env LOGZIO_LOGS_TOKEN="<LOGS-SHIPPING-TOKEN>" \
+-v /var/run/docker.sock:/var/run/docker.sock:ro \
+-v /var/lib/docker/containers:/var/lib/docker/containers \
+-e HEADERS="user-agent:logzio-docker-logs" \
+logzio/docker-logs-collector-<ARCH>:latest
+```
+
+#### Parameters
+
+| Parameter                   | Description                                                                                                                                                                                                                                                                                            |
+|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **LOGZIO_LOGS_TOKEN**            | **Required**. Your Logz.io account logs token. Replace `<LOGS-SHIPPING-TOKEN>` with the [token](https://app.logz.io/#/dashboard/settings/general) of the account you want to ship to.                                                                                                                             |
+| **LOGZIO_URL**              | **Default**: `https://listener.logz.io:8071`.<br> The URL to send logs to.                                                                                                                                                                                                                             |
+| **LOGZIO_TYPE**             | **Default**: Docker image name.<br> The log type you'll use with this Docker. This is shown in your logs under the `type` field in Kibana.                                                                                                                                                             |
+| **LOGZIO_REGION**           | **Default**: US region.<br> Logz.io region code to ship the logs to. This region code changes depending on the region your account is hosted in. For example, accounts in the EU region have region code `eu`.                                                                                         |
+| **MATCH_CONTAINER_NAME**    | Specify a container to collect logs from. If the container's name matches, its logs are shipped; otherwise, its logs are ignored. <br /> **Note**: This option cannot be used with SKIP_CONTAINER_NAMES. Use regular expressions to keep records that match a specific field.                          |
+| **SKIP_CONTAINER_NAMES**    | Comma-separated list of containers to ignore. If a container's name matches a name on this list, its logs are ignored; otherwise, its logs are shipped. <br /> **Note**: This option cannot be used with MATCH_CONTAINER_NAME. Use regular expressions to exclude records that match a specific field. |
+| **MATCH_IMAGE_NAME**        | Specify a image to collect logs from. If the image's name matches, its logs are shipped; otherwise, its logs are ignored. <br /> **Note**: This option cannot be used with SKIP_IMAGE_NAMES. Use regular expressions to keep records that match a specific field.                                      |
+| **SKIP_IMAGE_NAMES**        | Comma-separated list of images to ignore. If a image's name matches a name on this list, its logs are ignored; otherwise, its logs are shipped. <br /> **Note**: This option cannot be used with MATCH_IMAGE_NAME. Use regular expressions to exclude records that match a specific field.             |
+| **INCLUDE_LINE**            | Regular expression to match the lines that you want Fluent Bit to include.                                                                                                                                                                                                                             |
+| **EXCLUDE_LINES**           | Regular expression to match the lines that you want Fluent Bit to exclude.                                                                                                                                                                                                                             |
+| **ADDITIONAL_FIELDS**       | Include additional fields with every message sent, formatted as `"fieldName1:fieldValue1,fieldName2:fieldValue2"`.                                                                                                                                                                                     |
+| **SET_FIELDS**              | Set fields with every message sent, formatted as `"fieldName1:fieldValue1,fieldName2:fieldValue2"`.                                                                                                                                                                                                    |
+| **LOG_LEVEL**               | **Default** `info`. Set log level for Fluent Bit. Allowed values are: `debug`, `info`, `warning`, `error`.                                                                                                                                                                                             |
+| **MULTILINE_START_STATE_RULE** | Regular expression for the start state rule of multiline parsing. <br /> See [Fluent Bit's official documentation](https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/multiline-parsing#rules-definition) for further info.                                                        |
+| **MULTILINE_CUSTOM_RULES**     | Custom rules for multiline parsing, separated by semicolons `;`.                                                                                                                                                                                                                                       |
+| **READ_FROM_HEAD**         | **Default** `true`. Specify if Fluent Bit should read logs from the beginning.                                                                                                                                                                                                                         |
+| **OUTPUT_ID**               | **Default** `output_id`. Specify the output ID for Fluent Bit logs.                                                                                                                                                                                                                                    |
+| **HEADERS**                 | Custom headers for Fluent Bit logs.                                                                                                                                                                                                                                                                    |
+
+
+### 3. Check Logz.io for your logs
+
+Spin up your Docker containers if you haven’t done so already. Give your logs a few minutes to get from your system to your Logz.io account.
+
+### Change log
+
+- 0.1.0:
+  - Initial release using Fluent Bit.

configs/parser_multiline.conf

@@ -0,0 +1,18 @@
+
+[MULTILINE_PARSER]
+    name          multiline-regex
+    type          regex
+    flush_timeout 1000
+    #
+    # Regex rules for multiline parsing
+    # ---------------------------------
+    #
+    # configuration hints:
+    #
+    #  - first state always has the name: start_state
+    #  - every field in the rule must be inside double quotes
+    #
+    # rules |   state name  | regex pattern                  | next state
+    # ------|---------------|--------------------------------------------
+    rule      "start_state"   "/^\[.*\] .*/"  "cont"
+    rule      "cont"          "/^\.*/"                     "cont"

configs/parsers.conf

@@ -0,0 +1,10 @@
+[PARSER]
+    Name        docker
+    Format      json
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
+    Time_Keep   On
+    # Adjust these keys based on the structure of your logs
+    # This example assumes the logs are in JSON format and the time field is named "time"
+    Decode_Field_As json log
+    Decode_Field_As escaped log

configs/plugins.conf

@@ -0,0 +1,2 @@
+[PLUGINS]
+    Path /fluent-bit/plugins/out_logzio.so