1.3.0 (#255)

.fossa.yml

@@ -0,0 +1,5 @@
+version: 3
+server: https://app.fossa.com
+project:
+  id: "fluent-plugin-splunk-hec"
+  team: "TA-Automation"

.github/workflows/ci_build_test.yaml

@@ -194,3 +194,36 @@ jobs:
           	--splunk-password $CI_SPLUNK_PASSWORD \
             --nodes-count $MINIKUBE_NODE_COUNTS\
           	-p no:warnings -s -n auto
+  fossa-scan:
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: run fossa anlyze and create report
+        run: |
+          curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash
+          fossa analyze  --include-unused-deps --debug
+          fossa report attribution --format text > /tmp/THIRDPARTY
+        env:
+          FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
+      - name: upload THIRDPARTY file
+        uses: actions/upload-artifact@v2
+        with:
+          name: THIRDPARTY
+          path: /tmp/THIRDPARTY
+      - name: run fossa test
+        run: |
+          fossa test --debug
+        env:
+          FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
+  semgrep:
+    runs-on: ubuntu-latest
+    name: security-sast-semgrep
+    if: github.actor != 'dependabot[bot]'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Semgrep
+        id: semgrep
+        uses: returntocorp/semgrep-action@v1
+        with:
+          publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }}

Gemfile

@@ -4,6 +4,7 @@ source 'https://rubygems.org'
 
 group :test do
   gem 'simplecov', require: false
+  gem 'net-smtp', require: false
 end
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }

Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-splunk-hec (1.2.13)
-      fluentd (>= 1.4)
+    fluent-plugin-splunk-hec (1.3.0)
+      fluentd (>= 1.5)
       multi_json (~> 1.13)
       net-http-persistent (~> 4.0)
       openid_connect (~> 1.1.8)
@@ -11,9 +11,9 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.2.3)
-      activesupport (= 7.0.2.3)
-    activesupport (7.0.2.3)
+    activemodel (7.0.3)
+      activesupport (= 7.0.3)
+    activesupport (7.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -28,13 +28,14 @@ GEM
     cool.io (1.7.1)
     crack (0.4.5)
       rexml
+    digest (3.1.0)
     docile (1.4.0)
-    fluentd (1.14.6)
+    fluentd (1.15.1)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.5, < 3.0.0)
+      serverengine (>= 2.3.0, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
@@ -46,6 +47,7 @@ GEM
     httpclient (2.8.3)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
+    io-wait (0.2.1)
     json-jwt (1.13.0)
       activesupport (>= 4.2)
       aes_key_wrap
@@ -54,10 +56,17 @@ GEM
       mini_mime (>= 0.1.1)
     mini_mime (1.1.2)
     minitest (5.15.0)
-    msgpack (1.5.1)
+    msgpack (1.5.4)
     multi_json (1.15.0)
     net-http-persistent (4.0.1)
       connection_pool (~> 2.2)
+    net-protocol (0.1.2)
+      io-wait
+      timeout
+    net-smtp (0.3.1)
+      digest
+      net-protocol
+      timeout
     openid_connect (1.1.8)
       activemodel
       attr_required (>= 1.0.0)
@@ -71,7 +80,7 @@ GEM
     power_assert (2.0.1)
     prometheus-client (4.0.0)
     public_suffix (4.0.6)
-    rack (2.2.3)
+    rack (2.2.3.1)
     rack-oauth2 (1.19.0)
       activesupport
       attr_required
@@ -80,7 +89,7 @@ GEM
       rack (>= 2.1.0)
     rake (13.0.6)
     rexml (3.2.5)
-    serverengine (2.2.5)
+    serverengine (2.3.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.21.2)
@@ -96,14 +105,15 @@ GEM
       httpclient (>= 2.4)
     test-unit (3.5.3)
       power_assert
-    tzinfo (2.0.4)
+    timeout (0.2.0)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2022.1)
+    tzinfo-data (1.2022.2)
       tzinfo (>= 1.0.0)
     validate_email (0.1.6)
       activemodel (>= 3.0)
       mail (>= 2.2.5)
-    validate_url (1.0.13)
+    validate_url (1.0.15)
       activemodel (>= 3.0.0)
       public_suffix
     webfinger (1.2.0)
@@ -114,7 +124,7 @@ GEM
       crack (>= 0.3.2)
       hashdiff
     webrick (1.7.0)
-    yajl-ruby (1.4.2)
+    yajl-ruby (1.4.3)
 
 PLATFORMS
   ruby
@@ -123,6 +133,7 @@ DEPENDENCIES
   bundler (~> 2.0)
   fluent-plugin-splunk-hec!
   minitest (~> 5.0)
+  net-smtp
   rake (>= 12.0)
   simplecov
   test-unit (~> 3.0)

VERSION

@@ -1 +1 @@
-1.2.13
+1.3.0

docker/Gemfile

@@ -3,20 +3,21 @@ source 'https://rubygems.org'
 # This is separate gemfile for building docker image that has all plugins
 # for kubernetes log collection agent
 # List all required gems here and install via bundler to resolve dependencies
-gem "fluentd", ">=1.14.5"
+gem "fluentd", ">=1.15"
 gem "fluent-plugin-systemd", "=1.0.2"
 gem "fluent-plugin-concat", "=2.4.0"
 gem "fluent-plugin-prometheus", "=2.0.2"
 gem "fluent-plugin-jq", "=0.5.1"
-gem "fluent-plugin-kubernetes_metadata_filter", ">=2.9.2"
-gem "oj", "=3.11.2"
+gem 'kubeclient', git: 'https://github.com/splunk/kubeclient.git', ref: '955ec5b'
+gem 'fluent-plugin-kubernetes_metadata_filter', '~> 3.1'
+gem "oj", ">=3.11.2"
 gem 'multi_json', '~> 1.13'
 gem 'net-http-persistent', '~> 4.0'
 gem 'openid_connect', '~> 1.1.8'
 gem 'prometheus-client', '>= 2.1.0'
 gem 'activesupport', '~> 5.2.4.3'
 gem 'http_parser.rb', '=0.8.0'
-gem "rack", ">= 2.1.4"
+gem "rack", "=2.2.3.1"
 gem "fluent-plugin-record-modifier", ">=2.1"
 
 

docker/Gemfile.lock

@@ -1,8 +1,19 @@
+GIT
+  remote: https://github.com/splunk/kubeclient.git
+  revision: 955ec5b62f64b6065b25a7c0c76467e8b6badf17
+  ref: 955ec5b
+  specs:
+    kubeclient (4.9.3)
+      http (>= 3.0, < 5.0)
+      jsonpath (~> 1.0)
+      recursive-open-struct (~> 1.1, >= 1.1.1)
+      rest-client (~> 2.0)
+
 PATH
   remote: gem
   specs:
-    fluent-plugin-splunk-hec (1.2.13)
-      fluentd (>= 1.4)
+    fluent-plugin-splunk-hec (1.3.0)
+      fluentd (>= 1.5)
       multi_json (~> 1.13)
       net-http-persistent (~> 4.0)
       openid_connect (~> 1.1.8)
@@ -37,8 +48,8 @@ GEM
     fluent-plugin-jq (0.5.1)
       fluentd (>= 0.14.10, < 2)
       multi_json (~> 1.13)
-    fluent-plugin-kubernetes_metadata_filter (2.9.4)
-      fluentd (>= 0.14.0, < 1.15)
+    fluent-plugin-kubernetes_metadata_filter (3.1.0)
+      fluentd (>= 0.14.0, < 1.16)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
     fluent-plugin-prometheus (2.0.2)
@@ -49,12 +60,12 @@ GEM
     fluent-plugin-systemd (1.0.2)
       fluentd (>= 0.14.11, < 2)
       systemd-journal (~> 1.3.2)
-    fluentd (1.14.6)
+    fluentd (1.15.1)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.5, < 3.0.0)
+      serverengine (>= 2.3.0, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
@@ -82,11 +93,6 @@ GEM
       bindata
     jsonpath (1.1.0)
       multi_json
-    kubeclient (4.9.3)
-      http (>= 3.0, < 5.0)
-      jsonpath (~> 1.0)
-      recursive-open-struct (~> 1.1, >= 1.1.1)
-      rest-client (~> 2.0)
     lru_redux (1.1.0)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
@@ -95,7 +101,7 @@ GEM
     mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
     minitest (5.15.0)
-    msgpack (1.5.1)
+    msgpack (1.5.4)
     multi_json (1.15.0)
     net-http-persistent (4.0.0)
       connection_pool (~> 2.2)
@@ -113,7 +119,7 @@ GEM
       webfinger (>= 1.0.1)
     prometheus-client (2.1.0)
     public_suffix (4.0.6)
-    rack (2.2.3)
+    rack (2.2.3.1)
     rack-oauth2 (1.19.0)
       activesupport
       attr_required
@@ -127,7 +133,7 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    serverengine (2.2.5)
+    serverengine (2.3.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.5)
@@ -138,9 +144,9 @@ GEM
     systemd-journal (1.3.3)
       ffi (~> 1.9)
     thread_safe (0.3.6)
-    tzinfo (1.2.9)
+    tzinfo (1.2.10)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2022.1)
+    tzinfo-data (1.2022.2)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
@@ -155,7 +161,7 @@ GEM
       activesupport
       httpclient (>= 2.4)
     webrick (1.7.0)
-    yajl-ruby (1.4.2)
+    yajl-ruby (1.4.3)
 
 PLATFORMS
   ruby
@@ -164,19 +170,20 @@ DEPENDENCIES
   activesupport (~> 5.2.4.3)
   fluent-plugin-concat (= 2.4.0)
   fluent-plugin-jq (= 0.5.1)
-  fluent-plugin-kubernetes_metadata_filter (>= 2.9.2)
+  fluent-plugin-kubernetes_metadata_filter (~> 3.1)
   fluent-plugin-prometheus (= 2.0.2)
   fluent-plugin-record-modifier (>= 2.1)
   fluent-plugin-splunk-hec!
   fluent-plugin-systemd (= 1.0.2)
-  fluentd (>= 1.14.5)
+  fluentd (>= 1.15)
   http_parser.rb (= 0.8.0)
+  kubeclient!
   multi_json (~> 1.13)
   net-http-persistent (~> 4.0)
-  oj (= 3.11.2)
+  oj (>= 3.11.2)
   openid_connect (~> 1.1.8)
   prometheus-client (>= 2.1.0)
-  rack (>= 2.1.4)
+  rack (= 2.2.3.1)
 
 BUNDLED WITH
    2.3.11

fluent-plugin-splunk-hec.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.3.0'
 
-  spec.add_runtime_dependency 'fluentd', '>= 1.4'
+  spec.add_runtime_dependency 'fluentd', '>= 1.5'
   spec.add_runtime_dependency 'multi_json', '~> 1.13'
   spec.add_runtime_dependency 'net-http-persistent', '~> 4.0'
   spec.add_runtime_dependency 'openid_connect', '~> 1.1.8'

lib/fluent/plugin/out_splunk_hec.rb

@@ -9,6 +9,7 @@
 require 'openssl'
 require 'multi_json'
 require 'net/http/persistent'
+require 'zlib'
 
 module Fluent::Plugin
   class SplunkHecOutput < SplunkOutput
@@ -96,6 +97,9 @@ class SplunkHecOutput < SplunkOutput
     desc 'When set to true, all fields defined in `index_key`, `host_key`, `source_key`, `sourcetype_key`, `metric_name_key`, `metric_value_key` will not be removed from the original event.'
     config_param :keep_keys, :bool, default: false
 
+    desc 'Indicates if GZIP Compression is enabled.'
+    config_param :gzip_compression, :bool, default: false
+
     desc 'App name'
     config_param :app_name, :string, default: "hec_plugin_gem"
 
@@ -321,13 +325,20 @@ def new_connection
         c.override_headers['Authorization'] = "Splunk #{@hec_token}"
         c.override_headers['__splunk_app_name'] = "#{@app_name}"
         c.override_headers['__splunk_app_version'] = "#{@app_version}"
-
       end
     end
 
     def write_to_splunk(chunk)
       post = Net::HTTP::Post.new @api.request_uri
-      post.body = chunk.read
+      if @gzip_compression
+        post.add_field("Content-Encoding", "gzip")
+        gzip_stream = Zlib::GzipWriter.new StringIO.new
+        gzip_stream << chunk.read
+        post.body = gzip_stream.close.string
+      else
+        post.body = chunk.read
+      end
+
       log.debug { "[Sending] Chunk: #{dump_unique_id_hex(chunk.unique_id)}(#{post.body.bytesize}B)." }
       log.trace { "POST #{@api} body=#{post.body}" }
       begin