Merge branch 'fluent:master' into out-es-add-cloud-apikey

.github/workflows/build-legacy-branch.yaml

@@ -80,7 +80,7 @@ jobs:
 
       - name: Build the legacy x86_64 debug image
         if: matrix.arch == 'amd64'
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./Dockerfile.x86_64.debug
           context: .
@@ -102,7 +102,7 @@ jobs:
             raw,${{ matrix.suffix }}-${{ inputs.ref }}
 
       - name: Build the legacy ${{ matrix.arch }} image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./Dockerfile.${{ matrix.suffix }}
           context: .

.github/workflows/call-build-images.yaml

@@ -114,7 +114,7 @@ jobs:
 
       - name: Build the production images
         id: build_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -141,7 +141,7 @@ jobs:
 
       - name: Build the debug multi-arch images
         id: debug_build_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -314,7 +314,7 @@ jobs:
           docker push ${{ inputs.registry }}/${{ inputs.image }}:windows-${{ matrix.windows-base-version }}-${{ inputs.version }}
 
         # We cannot use this action as it requires privileged mode
-        # uses: docker/build-push-action@v5
+        # uses: docker/build-push-action@v6
         # with:
         #   file: ./dockerfiles/Dockerfile.windows
         #   context: .

.github/workflows/call-integration-image-build.yaml

@@ -62,7 +62,7 @@ jobs:
             raw,${{ inputs.image-tag }}
 
       - name: Build the AMD64 image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./dockerfiles/Dockerfile
           context: .
@@ -83,7 +83,7 @@ jobs:
             raw,${{ inputs.image-tag }}-debug
 
       - name: Build the AMD64 debug image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: ./dockerfiles/Dockerfile
           context: .

.github/workflows/pr-compile-check.yaml

@@ -22,7 +22,7 @@ jobs:
       uses: docker/setup-buildx-action@v3
 
     - name: Attempt to build current source for CentOS 7
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         context: .
         file: ./dockerfiles/Dockerfile.centos7

.github/workflows/pr-image-tests.yaml

@@ -36,7 +36,7 @@ jobs:
 
         - name: Build the multi-arch images
           id: build
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v6
           with:
             file: ./dockerfiles/Dockerfile
             context: .
@@ -55,7 +55,7 @@ jobs:
           shell: bash
 
         - name: Build the debug multi-arch images
-          uses: docker/build-push-action@v5
+          uses: docker/build-push-action@v6
           with:
             file: ./dockerfiles/Dockerfile
             context: .

.github/workflows/staging-release.yaml

@@ -67,14 +67,6 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v4
 
-    # Check we can download the AppVeyor build which confirms it matches the version to release as well as being a successful build
-    - name: Get Appveyor binaries
-      run: |
-        ./packaging/appveyor-download.sh
-      shell: bash
-      env:
-        TAG: v${{ github.event.inputs.version }}
-
   staging-release-generate-package-matrix:
     name: Get package matrix
     runs-on: ubuntu-latest
@@ -356,14 +348,6 @@ jobs:
         AWS_REGION: "us-east-1"
       shell: bash
 
-    - name: Get Appveyor binaries
-      run: |
-        ./packaging/appveyor-download.sh
-      shell: bash
-      env:
-        TAG: v${{ github.event.inputs.version }}
-        OUTPUT_DIR: appveyor
-
     - name: Move components from staging and setup
       run: |
         ./packaging/update-source-packages.sh
@@ -518,8 +502,8 @@ jobs:
           TAG: ${{ steps.get-tag.outputs.tag }}
 
   staging-release-images-latest-tags:
-    # Only update latest tags for 3.0 releases
-    if: startsWith(github.event.inputs.version, '3.0')
+    # Only update latest tags for 3.1 releases
+    if: startsWith(github.event.inputs.version, '3.1')
     name: Release latest Linux container images
     runs-on: ubuntu-latest
     needs:
@@ -807,21 +791,21 @@ jobs:
           target_commitish: '2.1'
           make_latest: false
 
-      - name: Release 2.2 - not latest
+      - name: Release 3.0 - not latest
         uses: softprops/action-gh-release@v2
-        if: startsWith(inputs.version, '2.2')
+        if: startsWith(inputs.version, '3.0')
         with:
           body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
           draft: false
           generate_release_notes: true
           name: "Fluent Bit ${{ inputs.version }}"
           tag_name: v${{ inputs.version }}
-          target_commitish: '2.2'
+          target_commitish: '3.0'
           make_latest: false
 
-      - name: Release 3.0 and latest
+      - name: Release 3.1 and latest
         uses: softprops/action-gh-release@v2
-        if: startsWith(inputs.version, '3.0')
+        if: startsWith(inputs.version, '3.1')
         with:
           body: "https://fluentbit.io/announcements/v${{ inputs.version }}/"
           draft: false
@@ -914,13 +898,20 @@ jobs:
           ref: 2.2
           token: ${{ secrets.GH_PA_TOKEN }}
 
-      - name: Release 3.0 and latest
+      - name: Release 3.0 - not latest
         if: startsWith(inputs.version, '3.0')
         uses: actions/checkout@v4
         with:
           repository: fluent/fluent-bit-docs
           token: ${{ secrets.GH_PA_TOKEN }}
 
+      - name: Release 3.1 and latest
+        if: startsWith(inputs.version, '3.1')
+        uses: actions/checkout@v4
+        with:
+          repository: fluent/fluent-bit-docs
+          token: ${{ secrets.GH_PA_TOKEN }}
+
       - name: Ensure we have the script we need
         run: |
           if [[ ! -f update-release-version-docs.sh ]] ; then
@@ -993,9 +984,15 @@ jobs:
         with:
           ref: 2.2
 
-      - name: Release 3.0 and latest
+      - name: Release 3.0 not latest
         if: startsWith(inputs.version, '3.0')
         uses: actions/checkout@v4
+        with:
+          ref: 3.0
+
+      - name: Release 3.1 latest
+        if: startsWith(inputs.version, '3.1')
+        uses: actions/checkout@v4
 
       # Get the new version to use
       - name: 'Get next minor version'

CMakeLists.txt

@@ -8,7 +8,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  3)
 set(FLB_VERSION_MINOR  1)
-set(FLB_VERSION_PATCH  0)
+set(FLB_VERSION_PATCH  4)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -110,6 +110,8 @@ option(FLB_RELEASE             "Build with release mode (-O2 -g -DNDEBUG)"   No)
 set(FLB_IPO "ReleaseOnly" CACHE STRING "Build with interprocedural optimization")
 set_property(CACHE FLB_IPO PROPERTY STRINGS "On;Off;ReleaseOnly")
 option(FLB_SMALL               "Optimise for small size"       No)
+set(FLB_SECURITY "ReleaseOnly" CACHE STRING "Build with security optimizations")
+set_property(CACHE FLB_SECURITY PROPERTY STRINGS "On;Off;ReleaseOnly")
 option(FLB_COVERAGE            "Build with code-coverage"      No)
 option(FLB_JEMALLOC            "Build with Jemalloc support"   No)
 option(FLB_REGEX               "Build with Regex support"     Yes)
@@ -283,7 +285,7 @@ endif ()
 find_package(Git)
 # If we do not have Git or this is not a Git repo or another error this just is ignored and we have no output at runtime.
 execute_process(COMMAND
-  "${GIT_EXECUTABLE}" log -1 --format=%H
+  "${GIT_EXECUTABLE}" -c log.showSignature=false log -1 --format=%H
   WORKING_DIRECTORY "${FLB_ROOT}"
   OUTPUT_VARIABLE FLB_GIT_HASH
   ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -332,6 +334,21 @@ if(FLB_IPO STREQUAL "On" OR (FLB_IPO STREQUAL "ReleaseOnly" AND FLB_RELEASE))
   endif()
 endif()
 
+# Harden release binary against security vulnerabilities
+if(FLB_SECURITY STREQUAL "On" OR (FLB_SECURITY STREQUAL "ReleaseOnly" AND FLB_RELEASE))
+  if (NOT MSVC)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,relro,-z,now")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack")
+    if(NOT FLB_SMALL)
+      set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector")
+      # Fortify requires optimization
+      if(CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FORTIFY_SOURCE=1")
+      endif()
+    endif()
+  endif()
+endif()
+
 if(FLB_PARSER)
   FLB_DEFINITION(FLB_HAVE_PARSER)
   message(STATUS "Enabling FLB_REGEX since FLB_PARSER requires")

cmake/libraries.cmake

@@ -10,7 +10,7 @@ set(FLB_PATH_LIB_MSGPACK     "lib/msgpack-c")
 set(FLB_PATH_LIB_NGHTTP2     "lib/nghttp2")
 set(FLB_PATH_LIB_AVRO        "lib/avro")
 set(FLB_PATH_LIB_CHUNKIO     "lib/chunkio")
-set(FLB_PATH_LIB_LUAJIT      "lib/luajit-3065c9")
+set(FLB_PATH_LIB_LUAJIT      "lib/luajit-04dca791")
 set(FLB_PATH_LIB_MONKEY      "lib/monkey")
 set(FLB_PATH_LIB_JSMN        "lib/jsmn")
 set(FLB_PATH_LIB_SQLITE      "lib/sqlite-amalgamation-3450200")
@@ -19,8 +19,8 @@ set(FLB_PATH_LIB_ONIGMO      "lib/onigmo")
 set(FLB_PATH_LIB_MPACK       "lib/mpack-amalgamation-1.1.1")
 set(FLB_PATH_LIB_MINIZ       "lib/miniz")
 set(FLB_PATH_LIB_TUTF8E      "lib/tutf8e")
-set(FLB_PATH_LIB_CARES       "lib/c-ares-1.24.0")
+set(FLB_PATH_LIB_CARES       "lib/c-ares-1.32.0")
 set(FLB_PATH_LIB_SNAPPY      "lib/snappy-fef67ac")
-set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.3.0")
+set(FLB_PATH_LIB_RDKAFKA     "lib/librdkafka-2.4.0")
 set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb")
 set(FLB_PATH_LIB_WASM_MICRO_RUNTIME  "lib/wasm-micro-runtime-WAMR-1.3.0")

cmake/luajit.cmake

@@ -1,4 +1,5 @@
 # luajit cmake
 option(LUAJIT_DIR "Path of LuaJIT 2.1 source dir" ON)
+option(LUAJIT_SETUP_INCLUDE_DIR "Setup include dir if parent is present" OFF)
 set(LUAJIT_DIR ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_LUAJIT})
 add_subdirectory("lib/luajit-cmake")

cmake/plugins_options.cmake

@@ -47,6 +47,7 @@ DEFINE_OPTION(FLB_IN_PROMETHEUS_SCRAPE        "Enable Prometheus Scrape input pl
 DEFINE_OPTION(FLB_IN_RANDOM                   "Enable random input plugin"                   ON)
 DEFINE_OPTION(FLB_IN_SERIAL                   "Enable Serial input plugin"                   ON)
 DEFINE_OPTION(FLB_IN_SPLUNK                   "Enable Splunk HTTP HEC input plugin"          ON)
+DEFINE_OPTION(FLB_IN_STATSD                   "Enable StatsD input plugin"                   ON)
 DEFINE_OPTION(FLB_IN_STDIN                    "Enable Standard input plugin"                 ON)
 DEFINE_OPTION(FLB_IN_STORAGE_BACKLOG          "Enable storage backlog input plugin"          ON)
 DEFINE_OPTION(FLB_IN_SYSLOG                   "Enable Syslog input plugin"                   ON)
@@ -67,7 +68,7 @@ DEFINE_OPTION(FLB_PROCESSOR_CONTENT_MODIFIER  "Enable content modifier processor
 DEFINE_OPTION(FLB_PROCESSOR_LABELS            "Enable metrics label manipulation processor"  ON)
 DEFINE_OPTION(FLB_PROCESSOR_METRICS_SELECTOR  "Enable metrics selector processor"            ON)
 DEFINE_OPTION(FLB_PROCESSOR_SQL               "Enable SQL processor"                         ON)
-
+DEFINE_OPTION(FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE "Enable OpenTelemetry envelope processor"  ON)
 
 # Filters
 # =======
@@ -87,12 +88,14 @@ DEFINE_OPTION(FLB_FILTER_MULTILINE            "Enable multiline filter"
 DEFINE_OPTION(FLB_FILTER_NEST                 "Enable nest filter"                           ON)
 DEFINE_OPTION(FLB_FILTER_NIGHTFALL            "Enable Nightfall filter"                      ON)
 DEFINE_OPTION(FLB_FILTER_PARSER               "Enable parser filter"                         ON)
+DEFINE_OPTION(FLB_FILTER_RECORD_MODIFIER      "Enable record_modifier filter"                ON)
 DEFINE_OPTION(FLB_FILTER_REWRITE_TAG          "Enable tag rewrite filter"                    ON)
 DEFINE_OPTION(FLB_FILTER_STDOUT               "Enable stdout filter"                         ON)
 DEFINE_OPTION(FLB_FILTER_SYSINFO              "Enable sysinfo filter"                        ON)
 DEFINE_OPTION(FLB_FILTER_THROTTLE             "Enable throttle filter"                       ON)
 DEFINE_OPTION(FLB_FILTER_THROTTLE_SIZE        "Enable throttle size filter"                  OFF)
 DEFINE_OPTION(FLB_FILTER_TYPE_CONVERTER       "Enable type converter filter"                 ON)
+DEFINE_OPTION(FLB_FILTER_TENSORFLOW           "Enable tensorflow filter"                     OFF)
 DEFINE_OPTION(FLB_FILTER_WASM                 "Enable WASM filter"                           ON)
 
 # Outputs (destinations)
@@ -119,8 +122,11 @@ DEFINE_OPTION(FLB_OUT_KAFKA                   "Enable Kafka output plugin"
 DEFINE_OPTION(FLB_OUT_KAFKA_REST              "Enable Kafka Rest output plugin"              ON)
 DEFINE_OPTION(FLB_OUT_KINESIS_FIREHOSE        "Enable AWS Firehose output plugin"            ON)
 DEFINE_OPTION(FLB_OUT_KINESIS_STREAMS         "Enable AWS Kinesis output plugin"             ON)
+DEFINE_OPTION(FLB_OUT_LIB                     "Enable library mode output plugin"            ON)
 DEFINE_OPTION(FLB_OUT_LOGDNA                  "Enable LogDNA output plugin"                  ON)
 DEFINE_OPTION(FLB_OUT_LOKI                    "Enable Loki output plugin"                    ON)
+DEFINE_OPTION(FLB_OUT_NATS                    "Enable NATS output plugin"                    ON)
+DEFINE_OPTION(FLB_OUT_NRLOGS                  "Enable New Relic output plugin"               ON)
 DEFINE_OPTION(FLB_OUT_NULL                    "Enable dev null output plugin"                ON)
 DEFINE_OPTION(FLB_OUT_OPENSEARCH              "Enable OpenSearch output plugin"              ON)
 DEFINE_OPTION(FLB_OUT_OPENTELEMETRY           "Enable OpenTelemetry plugin"                  ON)

cmake/windows-setup.cmake

@@ -7,13 +7,7 @@ if(FLB_WINDOWS_DEFAULTS)
   message(STATUS "Overriding setttings with windows-setup.cmake")
   set(FLB_REGEX                 Yes)
   set(FLB_BACKTRACE              No)
-  # LuaJIT does not currently support Windows ARM64 architecture so we disable it for now.
-  # See also: https://github.com/LuaJIT/LuaJIT/issues/593
-  if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ARM64|AARCH64)")
-    set(FLB_LUAJIT               No)
-  else()
-    set(FLB_LUAJIT              Yes)
-  endif()
+  set(FLB_LUAJIT                Yes)
   set(FLB_EXAMPLES              Yes)
   set(FLB_PARSER                Yes)
   set(FLB_TLS                   Yes)

conf/parsers.conf

@@ -128,3 +128,11 @@
     Name    kube-custom
     Format  regex
     Regex   (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$
+
+[PARSER]
+    # Examples: TCP: https://rubular.com/r/Q8YY6fHqlqwGI0  UDP: https://rubular.com/r/B0ID69H9FvN0tp
+    Name    kmsg-netfilter-log
+    Format  regex
+    Regex   ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) kernel - - - \[[0-9\.]*\] (?<logprefix>[^ ]*)\s?IN=(?<in>[^ ]*) OUT=(?<out>[^ ]*) MAC=(?<macsrc>[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}):(?<macdst>[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}):(?<ethtype>[0-9a-f]{2}:[0-9a-f]{2}) SRC=(?<saddr>[^ ]*) DST=(?<daddr>[^ ]*) LEN=(?<len>[^ ]*) TOS=(?<tos>[^ ]*) PREC=(?<prec>[^ ]*) TTL=(?<ttl>[^ ]*) ID=(?<id>[^ ]*) (D*F*)\s*PROTO=(?<proto>[^ ]*)\s?((SPT=)?(?<sport>[0-9]*))\s?((DPT=)?(?<dport>[0-9]*))\s?((LEN=)?(?<protolen>[0-9]*))\s?((WINDOW=)?(?<window>[0-9]*))\s?((RES=)?(?<res>0?x?[0-9]*))\s?(?<flag>[^ ]*)\s?((URGP=)?(?<urgp>[0-9]*))
+    Time_Key  time
+    Time_Format  %Y-%m-%dT%H:%M:%S.%L%z

dockerfiles/Dockerfile

@@ -11,7 +11,7 @@
 # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/
 
 # Set this to the current release version: it gets done so as part of the release.
-ARG RELEASE_VERSION=3.0.7
+ARG RELEASE_VERSION=3.1.4
 
 # For multi-arch builds - assumption is running on an AMD64 host
 FROM multiarch/qemu-user-static:x86_64-arm as qemu-arm32

dockerfiles/Dockerfile.centos7

@@ -4,7 +4,9 @@
 FROM centos:7
 
 # hadolint ignore=DL3032, DL3033
-RUN yum -y update && \
+RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \
+    sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \
+    yum -y update && \
     yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \
     wget unzip systemd-devel wget flex bison \
     cyrus-sasl-lib cyrus-sasl-devel openssl openss-libs openssl-devel \

fluent-bit-3.0.7.bb → fluent-bit-3.1.4.bb

@@ -16,7 +16,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93"
 SECTION = "net"
 
 PR = "r0"
-PV = "3.0.7"
+PV = "3.1.4"
 
 SRCREV = "v${PV}"
 SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1"

include/fluent-bit/flb_config_map.h

@@ -24,6 +24,7 @@
 #include <fluent-bit/flb_slist.h>
 #include <fluent-bit/flb_sds.h>
 #include <monkey/mk_core.h>
+#include <cfl/cfl.h>
 
 /* Configuration types */
 #define FLB_CONFIG_MAP_STR         0    /* string */
@@ -47,6 +48,8 @@
 #define FLB_CONFIG_MAP_SLIST_3   43   /* split up to 3 nodes + remaining data */
 #define FLB_CONFIG_MAP_SLIST_4   44   /* split up to 4 nodes + remaining data */
 
+#define FLB_CONFIG_MAP_VARIANT   50   /* variant that wraps a kvlist or array */
+
 #define FLB_CONFIG_MAP_MULT       1
 
 struct flb_config_map_val {
@@ -57,6 +60,7 @@ struct flb_config_map_val {
         size_t s_num;                 /* FLB_CONFIG_MAP_SIZE */
         flb_sds_t str;                /* FLB_CONFIG_MAP_STR */
         struct mk_list *list;         /* FLB_CONFIG_MAP_CLIST and FLB_CONFIG_MAP_SLIST */
+        struct cfl_variant *variant;  /* FLB_CONFIG_MAP_VARIANT */
     } val;
     struct mk_list *mult;
     struct mk_list _head;             /* Link to list if this entry is a 'multiple' entry */