Merge branch 'master' into zstd_comp_support

.github/workflows/scorecards.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
 
@@ -55,7 +55,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: SARIF file
           path: results.sarif
@@ -64,6 +64,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@2c779ab0d087cd7fe7b826087247c2c81f27bfa6 # v3.26.5
+        uses: github/codeql-action/upload-sarif@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
         with:
           sarif_file: results.sarif

.github/workflows/test-ruby-head.yml

@@ -19,9 +19,9 @@ jobs:
 
     name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a6e6f86333f0a2523ece813039b8b4be04560854 # v1.190.0
+        uses: ruby/setup-ruby@28c4deda893d5a96a6b2d958c5b47fc18d65c9d3 # v1.213.0
         with:
           ruby-version: ${{ matrix.ruby-version }}
       - name: Install addons

.github/workflows/test.yml

@@ -12,6 +12,10 @@ on:
       - '*.md'
       - 'lib/fluent/version.rb'
 
+concurrency:
+  group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 permissions: read-all
 
 jobs:
@@ -22,15 +26,16 @@ jobs:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
-        ruby-version: ['3.3', '3.2', '3.1', '3.0', '2.7']
+        ruby-version: ['3.4', '3.3', '3.2']
 
     name: Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Set up Ruby
-        uses: ruby/setup-ruby@a6e6f86333f0a2523ece813039b8b4be04560854 # v1.190.0
+        uses: ruby/setup-ruby@28c4deda893d5a96a6b2d958c5b47fc18d65c9d3 # v1.213.0
         with:
           ruby-version: ${{ matrix.ruby-version }}
+          rubygems: latest
       - name: Install addons
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: sudo apt-get install libgmp3-dev libcap-ng-dev

CHANGELOG.md

@@ -1,3 +1,57 @@
+# v1.18
+
+## Release v1.18.0 - 2024/11/29
+
+### Enhancement
+
+* Add zero-downtime-restart feature for non-Windows
+  https://github.com/fluent/fluentd/pull/4624
+* Add with-source-only feature
+  https://github.com/fluent/fluentd/pull/4661
+  * `fluentd` command: Add `--with-source-only` option
+  * System configuration: Add `with_source_only` option
+* Embedded plugin: Add `out_buffer` plugin, which can be used for buffering and relabeling events
+  https://github.com/fluent/fluentd/pull/4661
+* Config File Syntax: Extend Embedded Ruby Code support for Hashes and Arrays
+  https://github.com/fluent/fluentd/pull/4580
+  * Example: `key {"foo":"#{1 + 1}"} => key {"foo":"2"}`
+  * Please note that this is not backward compatible, although we assume that this will never affect to actual existing configs.
+  * In case the behavior changes unintentionally, you can disable this feature by surrounding the entire value with single quotes.
+    * `key '{"foo":"#{1 + 1}"}' => key {"foo":"#{1 + 1}"}`
+* transport tls: Use SSL_VERIFY_NONE by default
+  https://github.com/fluent/fluentd/pull/4718
+* transport tls: Add ensure_fips option to ensure FIPS compliant mode
+  https://github.com/fluent/fluentd/pull/4720
+* plugin_helper/server: Add receive_buffer_size parameter in transport section
+  https://github.com/fluent/fluentd/pull/4649
+* filter_parser: Now able to handle multiple parsed results
+  https://github.com/fluent/fluentd/pull/4620
+* in_http: add `add_tag_prefix` option
+  https://github.com/fluent/fluentd/pull/4655
+* System configuration: add `path` option in `log` section
+  https://github.com/fluent/fluentd/pull/4604
+
+### Bug Fix
+
+* command: fix NoMethodError of --daemon under Windows
+  https://github.com/fluent/fluentd/pull/4716
+* `fluentd` command: fix `--plugin` (`-p`) option not to overwrite default value
+  https://github.com/fluent/fluentd/pull/4605
+
+### Misc
+
+* http_server: Ready to support Async 2.0 gem
+  https://github.com/fluent/fluentd/pull/4619
+* Minor code refactoring
+  * https://github.com/fluent/fluentd/pull/4641
+* CI fixes
+  * https://github.com/fluent/fluentd/pull/4638
+  * https://github.com/fluent/fluentd/pull/4644
+  * https://github.com/fluent/fluentd/pull/4675
+  * https://github.com/fluent/fluentd/pull/4676
+  * https://github.com/fluent/fluentd/pull/4677
+  * https://github.com/fluent/fluentd/pull/4686
+
 # v1.17
 
 ## Release v1.17.1 - 2024/08/19

README.md

@@ -30,7 +30,7 @@ Fluentd: Open-Source Log Collector
 
 ### Prerequisites
 
-- Ruby 2.7 or later
+- Ruby 3.2 or later
 - git
 
 `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
@@ -72,5 +72,3 @@ See [SECURITY](SECURITY.md) to contact us about vulnerability.
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
-
-[<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />](https://github.com/fluent/fluentd)

SECURITY.md

@@ -4,7 +4,8 @@
 
 | Version   | Supported          |
 | -------   | ------------------ |
-| 1.17.x    | :white_check_mark: |
+| 1.18.x    | :white_check_mark: |
+| 1.17.x    | :x:                |
 | 1.16.x    | :white_check_mark: |
 | <= 1.15.x | :x:                |
 

fluentd.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.license = "Apache-2.0"
 
-  gem.required_ruby_version = '>= 2.7'
+  gem.required_ruby_version = '>= 3.2'
 
   gem.add_runtime_dependency("bundler")
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
@@ -41,7 +41,9 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("drb", ["~> 2.2"])
 
   # gems that aren't default gems as of Ruby 3.5
-  gem.add_runtime_dependency("logger", ["~> 1.6"])
+  # logger 1.6.3 or later cause bug on windows,
+  # hold on 1.6.2 for a while. see https://github.com/ruby/logger/issues/107
+  gem.add_runtime_dependency("logger", ["1.6.2"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
@@ -64,8 +66,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("test-unit", ["~> 3.3"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", [">= 2.14", "< 4"])
-  gem.add_development_dependency("async", "~> 1.23")
-  gem.add_development_dependency("async-http", ">= 0.50.0")
+  gem.add_development_dependency("async-http", "~> 0.86")
   gem.add_development_dependency("aws-sigv4", ["~> 1.8"])
   gem.add_development_dependency("aws-sdk-core", ["~> 3.191"])
   gem.add_development_dependency("rexml", ["~> 3.2"])

lib/fluent/command/fluentd.rb

@@ -127,6 +127,12 @@
   cmd_opts[:without_source] = b
 }
 
+unless Fluent.windows?
+  op.on('--with-source-only', "Invoke a fluentd only with input plugins. The data is stored in a temporary buffer. Send SIGWINCH to cancel this mode and process the data (Not supported on Windows).", TrueClass) {|b|
+    cmd_opts[:with_source_only] = b
+  }
+end
+
 op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
   if (s == 'yaml') || (s == 'yml')
     cmd_opts[:config_file_type] = s.to_sym

lib/fluent/config/literal_parser.rb

@@ -254,8 +254,15 @@ def scan_json(is_array)
               buffer << line_buffer + "\n"
               line_buffer = ""
             else
-              # '#' is a char in json string
-              line_buffer << char
+              if @ss.exist?(/^\{[^}]+\}/)
+                # if it's interpolated string
+                skip(/\{/)
+                line_buffer << eval_embedded_code(scan_embedded_code)
+                skip(/\}/)
+              else
+                # '#' is a char in json string
+                line_buffer << char
+              end
             end
 
             next # This char '#' MUST NOT terminate json object.

lib/fluent/engine.rb

@@ -43,13 +43,15 @@ def initialize
       @system_config = SystemConfig.new
 
       @supervisor_mode = false
+
+      @root_agent_mutex = Mutex.new
     end
 
     MAINLOOP_SLEEP_INTERVAL = 0.3
 
     attr_reader :root_agent, :system_config, :supervisor_mode
 
-    def init(system_config, supervisor_mode: false)
+    def init(system_config, supervisor_mode: false, start_in_parallel: false)
       @system_config = system_config
       @supervisor_mode = supervisor_mode
 
@@ -58,7 +60,7 @@ def init(system_config, supervisor_mode: false)
 
       @log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
 
-      @root_agent = RootAgent.new(log: log, system_config: @system_config)
+      @root_agent = RootAgent.new(log: log, system_config: @system_config, start_in_parallel: start_in_parallel)
 
       self
     end
@@ -133,7 +135,15 @@ def emit_stream(tag, es)
     end
 
     def flush!
-      @root_agent.flush!
+      @root_agent_mutex.synchronize do
+        @root_agent.flush!
+      end
+    end
+
+    def cancel_source_only!
+      @root_agent_mutex.synchronize do
+        @root_agent.cancel_source_only!
+      end
     end
 
     def now
@@ -144,7 +154,9 @@ def now
     def run
       begin
         $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
-        start
+        @root_agent_mutex.synchronize do
+          start
+        end
 
         @fluent_log_event_router.start
 
@@ -158,47 +170,51 @@ def run
         raise
       end
 
-      stop_phase(@root_agent)
+      @root_agent_mutex.synchronize do
+        stop_phase(@root_agent)
+      end
     end
 
     # @param conf [Fluent::Config]
     # @param supervisor [Bool]
     # @reutrn nil
     def reload_config(conf, supervisor: false)
-      # configure first to reduce down time while restarting
-      new_agent = RootAgent.new(log: log, system_config: @system_config)
-      ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
-
-      ret.all_plugins.each do |plugin|
-        if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
-          raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
+      @root_agent_mutex.synchronize do
+        # configure first to reduce down time while restarting
+        new_agent = RootAgent.new(log: log, system_config: @system_config)
+        ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
+
+        ret.all_plugins.each do |plugin|
+          if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
+            raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
+          end
         end
-      end
 
-      # Assign @root_agent to new root_agent
-      # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
-      old_agent, @root_agent = @root_agent, new_agent
-      begin
-        @root_agent.configure(conf)
-      rescue
-        @root_agent = old_agent
-        raise
-      end
+        # Assign @root_agent to new root_agent
+        # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
+        old_agent, @root_agent = @root_agent, new_agent
+        begin
+          @root_agent.configure(conf)
+        rescue
+          @root_agent = old_agent
+          raise
+        end
 
-      unless @suppress_config_dump
-        $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
-      end
+        unless @suppress_config_dump
+          $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
+        end
 
-      # supervisor doesn't handle actual data. so the following code is unnecessary.
-      if supervisor
-        old_agent.shutdown      # to close thread created in #configure
-        return
-      end
+        # supervisor doesn't handle actual data. so the following code is unnecessary.
+        if supervisor
+          old_agent.shutdown      # to close thread created in #configure
+          return
+        end
 
-      stop_phase(old_agent)
+        stop_phase(old_agent)
 
-      $log.info 'restart fluentd worker', worker: worker_id
-      start_phase(new_agent)
+        $log.info 'restart fluentd worker', worker: worker_id
+        start_phase(new_agent)
+      end
     end
 
     def stop

lib/fluent/env.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'securerandom'
+
 require 'serverengine/utils'
 require 'fluent/oj_options'
 
@@ -25,6 +27,7 @@ module Fluent
   DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644
+  INSTANCE_ID = ENV['FLUENT_INSTANCE_ID'] || SecureRandom.uuid
 
   def self.windows?
     ServerEngine.windows?

lib/fluent/event_router.rb

@@ -286,7 +286,7 @@ def reset_optimization
 
     def find(tag)
       pipeline = nil
-      @match_rules.each_with_index { |rule, i|
+      @match_rules.each do |rule|
         if rule.match?(tag)
           if rule.collector.is_a?(Plugin::Filter)
             pipeline ||= Pipeline.new
@@ -301,7 +301,7 @@ def find(tag)
             return pipeline
           end
         end
-      }
+      end
 
       if pipeline
         # filter is matched but no match