diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 8285209..cacc9ae 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -72,7 +72,7 @@ jobs: # Build Docker Image - name: Build Docker Image run: | - docker build -f Dockerfile.amd64 -t logzio/docker-logs-collector-amd64:latest . + docker buildx build --platform linux/amd64 --load -t logzio/docker-logs-collector:amd64-test . # Install Docker Compose - name: Install Docker Compose diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 127214e..757ea07 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,12 +6,8 @@ on: jobs: build-and-push-images: - name: Build and Push Docker Images + name: Build and Push Multi-Arch Docker Image runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - architecture: [amd64, arm64] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -28,16 +24,15 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push image + - name: Build and push multi-arch image uses: docker/build-push-action@v5 with: context: . - file: Dockerfile.${{ matrix.architecture }} - platforms: linux/${{ matrix.architecture }} push: true + platforms: linux/amd64,linux/arm64 tags: | - logzio/docker-logs-collector-${{ matrix.architecture }}:latest - logzio/docker-logs-collector-${{ matrix.architecture }}:${{ github.ref_name }} + logzio/docker-logs-collector:latest + logzio/docker-logs-collector:${{ github.ref_name }} - name: Logout from Docker Hub if: always() diff --git a/Dockerfile.amd64 b/Dockerfile similarity index 51% rename from Dockerfile.amd64 rename to Dockerfile index bfe0a18..7ea3625 100644 --- a/Dockerfile.amd64 +++ b/Dockerfile @@ -1,6 +1,6 @@ -# Dockerfile.amd64 +# syntax=docker/dockerfile:1 -FROM python:3.12.4-slim AS base +FROM python:3.12.4-slim-bullseye AS base # Install dependencies using apt-get RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -15,10 +15,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gdb \ && rm -rf /var/lib/apt/lists/* -# Create the plugins directory and download the Logz.io plugin +# Define build argument for architecture +ARG TARGETARCH + +# Set the plugin URL based on the architecture +ENV LOGZIO_PLUGIN_URL_AMD64=https://github.com/logzio/fluent-bit-logzio-output/raw/master/build/out_logzio-linux.so +ENV LOGZIO_PLUGIN_URL_ARM64=https://github.com/logzio/fluent-bit-logzio-output/raw/master/build/out_logzio-linux-arm64.so + +# Determine the correct plugin URL based on TARGETARCH RUN mkdir -p /fluent-bit/plugins && \ - wget -O /fluent-bit/plugins/out_logzio.so \ - https://github.com/logzio/fluent-bit-logzio-output/raw/master/build/out_logzio-linux.so + if [ "$TARGETARCH" = "amd64" ]; then \ + export LOGZIO_PLUGIN_URL=$LOGZIO_PLUGIN_URL_AMD64; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + export LOGZIO_PLUGIN_URL=$LOGZIO_PLUGIN_URL_ARM64; \ + else \ + echo "Unsupported architecture: $TARGETARCH"; exit 1; \ + fi && \ + wget -O /fluent-bit/plugins/out_logzio.so $LOGZIO_PLUGIN_URL # Set working directory WORKDIR /opt/fluent-bit @@ -31,9 +44,9 @@ COPY docker-metadata.lua /fluent-bit/etc/docker-metadata.lua COPY create_fluent_bit_config.py /opt/fluent-bit/docker-collector-logs/create_fluent_bit_config.py # Use official Fluent Bit image for Fluent Bit binaries -FROM fluent/fluent-bit:3.1.4 AS fluent-bit +FROM fluent/fluent-bit:1.9.10 AS fluent-bit -# Copy Fluent Bit binary and plugins.conf to the base image +# Copy Fluent Bit binary to the base image FROM base COPY --from=fluent-bit /fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit @@ -42,4 +55,4 @@ COPY start.sh /start.sh RUN chmod +x /start.sh # Set the entrypoint to run the shell script -ENTRYPOINT ["/start.sh"] \ No newline at end of file +ENTRYPOINT ["/start.sh"] diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 deleted file mode 100644 index f2b8a06..0000000 --- a/Dockerfile.arm64 +++ /dev/null @@ -1,46 +0,0 @@ -# Dockerfile.arm64 - -FROM python:3.12.4-slim AS base - -# Install dependencies using apt-get -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - bash \ - libyaml-dev \ - libsystemd-dev \ - libsasl2-dev \ - libpq-dev \ - openssl \ - libssl-dev \ - gdb \ - && rm -rf /var/lib/apt/lists/* - - -# Create the plugins directory and download the Logz.io plugin -RUN mkdir -p /fluent-bit/plugins && \ - wget -O /fluent-bit/plugins/out_logzio.so \ - https://github.com/logzio/fluent-bit-logzio-output/raw/master/build/out_logzio-linux-arm64.so - -# Set working directory -WORKDIR /opt/fluent-bit - -# Copy configuration files and Lua script -COPY configs/parser_multiline.conf /fluent-bit/etc/parsers_multiline.conf -COPY configs/parsers.conf /fluent-bit/etc/parsers.conf -COPY configs/plugins.conf /fluent-bit/etc/plugins.conf -COPY docker-metadata.lua /fluent-bit/etc/docker-metadata.lua -COPY create_fluent_bit_config.py /opt/fluent-bit/docker-collector-logs/create_fluent_bit_config.py - -# Use official Fluent Bit image for Fluent Bit binaries -FROM fluent/fluent-bit:3.1.4 AS fluent-bit - -# Copy Fluent Bit binary and plugins.conf to the base image -FROM base -COPY --from=fluent-bit /fluent-bit/bin/fluent-bit /usr/local/bin/fluent-bit - -# Copy entrypoint script -COPY start.sh /start.sh -RUN chmod +x /start.sh - -# Set the entrypoint to run the shell script -ENTRYPOINT ["/start.sh"] \ No newline at end of file diff --git a/README.md b/README.md index 9a32990..e95806e 100644 --- a/README.md +++ b/README.md @@ -19,17 +19,11 @@ If you want to ship metrics to Logz.io, see [docker-collector-metrics](https://g Download the appropriate Docker image for your architecture (amd64 or arm64): ```shell -# For amd64 architecture -docker pull logzio/docker-logs-collector-amd64:latest - -# For arm64 architecture -docker pull logzio/docker-logs-collector-arm64:latest +docker pull logzio/docker-logs-collector:latest ``` ### 2. Run the container -Replace `` in the image name with either amd64 or arm64 based on your system architecture. - For a complete list of options, see the parameters below the code block.👇 ```shell @@ -38,7 +32,7 @@ docker run --name docker-logs-collector \ -v /var/run/docker.sock:/var/run/docker.sock:ro \ -v /var/lib/docker/containers:/var/lib/docker/containers \ -e HEADERS="user-agent:logzio-docker-logs" \ -logzio/docker-logs-collector-:latest +logzio/docker-logs-collector:latest ``` #### Parameters diff --git a/create_fluent_bit_config.py b/create_fluent_bit_config.py index f2bfdbe..4ac8049 100644 --- a/create_fluent_bit_config.py +++ b/create_fluent_bit_config.py @@ -165,33 +165,24 @@ def generate_filters(config): return filters def _get_modify_filters(config): - filters = "" - modify_filters_added = False - - # Add additional fields if specified - if config.additional_fields: - fields = config.additional_fields.split(',') - filters += """ + filters = """ [FILTER] Name modify Match * + Rename log message """ + # Add additional fields if specified + if config.additional_fields: + fields = config.additional_fields.split(',') for field in fields: try: key, value = field.split(':', 1) filters += f" Add {key.strip()} {value.strip()}\n" except ValueError: print(f"Warning: Skipping invalid additional field '{field}'. Expected format 'key:value'.") - modify_filters_added = True # Add set fields if specified if config.set_fields: - if not modify_filters_added: - filters += """ -[FILTER] - Name modify - Match * -""" fields = config.set_fields.split(',') for field in fields: try: @@ -199,6 +190,7 @@ def _get_modify_filters(config): filters += f" Set {key.strip()} {value.strip()}\n" except ValueError: print(f"Warning: Skipping invalid set field '{field}'. Expected format 'key:value'.") + return filters def _get_output_config(config): @@ -210,7 +202,7 @@ def _get_output_config(config): logzio_url {config.logzio_url} logzio_type {config.logzio_type} id {config.output_id} - headers user-agent:logzio-docker-fluentbit-logs + headers user-agent:logzio-docker-collector-logs """ if config.headers: output_config += f" headers {config.headers}\n" diff --git a/docker-compose.yml b/docker-compose.yml index fc7630b..3a4df28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: LOG_LEVEL: "info" docker-logs-collector: - image: logzio/docker-logs-collector-amd64:latest + image: logzio/docker-logs-collector:amd64-test container_name: docker-logs-collector environment: - LOGZIO_LOGS_TOKEN=${LOGZIO_LOGS_TOKEN} diff --git a/docker-metadata.lua b/docker-metadata.lua index a18f1b2..9971a48 100644 --- a/docker-metadata.lua +++ b/docker-metadata.lua @@ -105,8 +105,6 @@ function M.enrich_with_docker_metadata(tag, timestamp, record) local new_record = record new_record['docker_container_id'] = container_id - new_record['message'] = record.log - new_record['log'] = nil local cached_data = M.cache[container_id] if cached_data == nil or (current_time - cached_data['time'] > M.CACHE_TTL_SEC) then