Merge pull request #68 from 24367dfa/main

convert ruby deps to Gemfile

.github/workflows/ci.yaml

@@ -13,6 +13,7 @@ jobs:
         fluentd:
           - v1.15-staging
           - v1.15-ruby3
+          - v1.16
 
     steps:
       - name: Checkout

v1.16/Dockerfile

@@ -0,0 +1,55 @@
+FROM alpine:3.18
+
+ENV FLUENTD_VERSION 1.16.2
+
+LABEL Description="Fluentd docker image" Vendor="Kube logging" Version="${FLUENTD_VERSION}"
+
+# Do not split this into multiple RUN!
+# Docker creates a layer for every RUN-Statement
+# therefore an 'apk delete' has no effect
+RUN apk update \
+ && apk add --no-cache \
+        ca-certificates \
+        ruby ruby-irb ruby-etc \
+        tini libmaxminddb geoip \
+        snappy \
+ && buildDeps=" \
+      make gcc g++ libc-dev \
+      wget bzip2 zlib-dev git linux-headers \
+      automake autoconf libtool build-base \
+      ruby-dev libc6-compat geoip-dev \
+      snappy-dev \
+    " \
+ && apk add --no-cache --virtual .build-deps \
+        build-base \
+        ruby-dev gnupg \
+ && apk add $buildDeps \
+ && echo 'gem: --no-document' >> /etc/gemrc
+ADD ./Gemfile /Gemfile
+RUN gem install bundler && bundle install 
+RUN apk del .build-deps $buildDeps \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test
+
+RUN addgroup -S fluent && adduser -S -G fluent fluent \
+    # for log storage (maybe shared with host)
+    && mkdir -p /fluentd/log \
+    # configuration/plugins path (default: copied from .)
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    && chown -R fluent /fluentd && chgrp -R fluent /fluentd \
+    && mkdir -p /buffers && chown -R fluent /buffers
+
+
+COPY fluent.conf /fluentd/etc/
+COPY entrypoint.sh /bin/
+COPY healthy.sh /bin/
+
+
+ENV FLUENTD_CONF="fluent.conf"
+ENV RUBYLIB="/usr/lib/ruby/gems/3.1.0/gems/resolv-0.2.2/lib"
+
+ENV LD_PRELOAD=""
+EXPOSE 24224 5140
+
+USER fluent
+ENTRYPOINT ["tini",  "--", "/bin/entrypoint.sh"]
+CMD ["fluentd"]

v1.16/Gemfile

@@ -0,0 +1,64 @@
+source 'https://rubygems.org'
+
+gem 'oj', '3.14.2'
+gem 'activesupport', '6.1.7'
+gem 'http_parser.rb', '0.8.0'
+gem 'tzinfo', '>= 1.2.11'
+gem 'resolv', '0.2.2'
+gem 'json', '2.6.3'
+gem 'yajl-ruby', '1.4.1'
+gem 'async', '1.31.0'
+gem 'async-http', '0.60.1'
+gem 'fluentd', '1.16.2'
+gem 'prometheus-client', '4.1.0'
+gem 'bigdecimal', '3.1.4'
+gem 'webrick', '1.8.1'
+gem 'gelf', '3.1.0'
+gem 'elasticsearch', '8.6.0'
+gem 'elasticsearch-xpack', '7.17.7'
+gem 'snappy', '0.2.0'
+gem 'fluent-plugin-detect-exceptions', '0.0.15'
+gem 'fluent-plugin-remote-syslog', '1.1.0'
+gem 'fluent-plugin-webhdfs', '1.5.0'
+gem 'fluent-plugin-prometheus', '2.0.3'
+gem 'fluent-plugin-s3', '1.7.2'
+gem 'fluent-plugin-rewrite-tag-filter', '2.4.0'
+# gem 'fluent-plugin-azure-storage-append-blob', '0.2.1'
+gem 'fluent-plugin-aliyun-oss', '0.0.1'
+gem 'fluent-plugin-dedot_filter', '1.0.0'
+gem 'fluent-plugin-sumologic_output', '1.8.0'
+gem 'fluent-plugin-kafka', '0.18.1'
+gem 'fluent-plugin-geoip', '1.3.2'
+gem 'fluent-plugin-tag-normaliser', '0.1.2'
+gem 'fluent-plugin-grafana-loki', '1.2.20'
+gem 'fluent-plugin-concat', '2.5.0'
+gem 'fluent-plugin-kinesis', '3.4.2'
+gem 'fluent-plugin-parser-logfmt', '0.0.0'
+gem 'fluent-plugin-multi-format-parser', '1.0.0'
+gem 'fluent-plugin-record-modifier', '2.1.1'
+gem 'fluent-plugin-splunk-hec', '1.3.2'
+gem 'fluent-plugin-elasticsearch', '5.3.0'
+gem 'fluent-plugin-newrelic', '1.2.2'
+gem 'fluent-plugin-cloudwatch-logs', '0.14.3'
+gem 'fluent-plugin-opensearch', '1.1.2'
+# gem 'fluent-plugin-aws-elasticsearch-service', '2.4.1'
+gem 'fluent-plugin-kube-events-timestamp', '0.1.3'
+gem 'fluent-plugin-grok-parser', '2.6.2'
+gem 'fluent-plugin-throttle', '0.0.5'
+# gem 'fluent-plugin-logdna', '~> 0.4.0'
+gem 'fluent-plugin-datadog', '0.14.2'
+gem 'fluent-plugin-redis', '0.3.5'
+gem 'fluent-plugin-gelf-hs', '1.0.8'
+gem 'fluent-plugin-sqs', '3.0.0'
+gem 'fluent-plugin-mattermost', '0.2.2'
+gem 'fluent-plugin-vmware-loginsight', '1.4.0'
+gem 'fluent-plugin-vmware-log-intelligence', '2.0.6'
+gem 'fluent-plugin-mysqlslowquery', '0.0.9'
+gem 'fluent-plugin-logzio', '0.0.22'
+gem 'fluent-plugin-syslog_rfc5424', :git => 'https://github.com/acquia/fluent-plugin-syslog_rfc5424.git', :tag => '0.2.1'
+gem 'fluent-plugin-gcs', :git => 'https://github.com/kube-logging/fluent-plugin-gcs.git', :branch => 'master'
+gem 'fluent-plugin-label-router', :git => 'https://github.com/kube-logging/fluent-plugin-label-router.git', :ref => '2ff43789f895735b7852e0a1a809280e22d9e8ef'
+gem 'fluent-plugin-kubernetes-metadata-filter', :git => 'https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git', :tag => 'v1.16.2-sumo-0', :glob => 'fluent-plugin-kubernetes-metadata-filter/*.gemspec'
+gem 'fluent-plugin-enhance-k8s-metadata', :git => 'https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git', :tag => 'v1.16.2-sumo-0', :glob => 'fluent-plugin-enhance-k8s-metadata/*.gemspec'
+gem 'fluent-plugin-kubernetes-sumologic', :git => 'https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git', :tag => 'v1.16.2-sumo-0', :glob => 'fluent-plugin-kubernetes-sumologic/*.gemspec'
+gem 'kubeclient', :git => 'https://github.com/abonas/kubeclient.git', :tag => 'v4.11.0'

v1.16/entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+#source vars if file exists
+DEFAULT=/etc/default/fluentd
+
+if [ -r $DEFAULT ]; then
+    set -o allexport
+    . $DEFAULT
+    set +o allexport
+fi
+
+# If the user has supplied only arguments append them to `fluentd` command
+if [ "${1#-}" != "$1" ]; then
+    set -- fluentd "$@"
+fi
+
+# If user does not supply config file or plugins, use the default
+if [ "$1" = "fluentd" ]; then
+    if ! echo $@ | grep ' \-c' ; then
+       set -- "$@" -c /fluentd/etc/${FLUENTD_CONF}
+    fi
+
+    if ! echo $@ | grep ' \-p' ; then
+       set -- "$@" -p /fluentd/plugins
+    fi
+fi
+
+exec "$@"

v1.16/fluent.conf

@@ -0,0 +1,8 @@
+# This is the root config file, which only includes components of the actual configuration
+
+# Do not collect fluentd's own logs to avoid infinite loops.
+<match fluent.**>
+  @type null
+</match>
+
+@include /fluentd/etc/conf.d/*.conf

v1.16/healthy.sh

@@ -0,0 +1,23 @@
+#!/bin/sh -x
+
+# Liveness probe is aimed to help in situations where fluentd
+# silently hangs for no apparent reasons until manual restart.
+# The idea of this probe is that if fluentd is not queueing or
+# flushing chunks for 5 minutes, something is not right. If
+# you want to change the fluentd configuration, reducing amount of
+# logs fluentd collects, consider changing the threshold or turning
+# liveness probe off completely.
+# soiurce https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml#L58
+
+BUFFER_PATH=${BUFFER_PATH};
+LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300};
+
+if [ ! -e ${BUFFER_PATH} ];
+then
+  exit 1;
+fi;
+touch -d "@$(($(date +%s) - $LIVENESS_THRESHOLD_SECONDS))" /tmp/marker-liveness;
+if [ -z "$(find ${BUFFER_PATH} -type d -newer /tmp/marker-liveness -print -quit)" ];
+then
+  exit 1;
+fi;