Merge branch 'fluent:master' into master

.github/workflows/unit-tests.yaml

@@ -127,7 +127,7 @@ jobs:
         config:
           - name: "Aarch64 actuated testing"
             flb_option: "-DFLB_WITHOUT_flb-it-network=1 -DFLB_WITHOUT_flb-it-fstore=1"
-            omit_option: "-DFLB_WITHOUT_flb-it-utils=1 -DFLB_WITHOUT_flb-it-pack=1"
+            omit_option: ""
             global_option: "-DFLB_BACKTRACE=Off -DFLB_SHARED_LIB=Off -DFLB_DEBUG=On -DFLB_ALL=On -DFLB_EXAMPLES=Off"
             unit_test_option: "-DFLB_TESTS_INTERNAL=On"
             compiler: gcc

CMakeLists.txt

@@ -8,7 +8,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  3)
 set(FLB_VERSION_MINOR  2)
-set(FLB_VERSION_PATCH  1)
+set(FLB_VERSION_PATCH  3)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -30,6 +30,10 @@ endif()
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")
   set(FLB_SYSTEM_LINUX On)
   add_definitions(-DFLB_SYSTEM_LINUX)
+  if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)")
+    set(FLB_LINUX_ON_AARCH64 On)
+    add_definitions(-DFLB_LINUX_ON_AARCH64)
+  endif()
 endif()
 
 # Update CFLAGS
@@ -256,6 +260,8 @@ if(FLB_ALL)
 endif()
 
 if(FLB_DEV)
+  FLB_DEFINITION(FLB_HAVE_DEV)
+
   set(FLB_DEBUG             On)
   set(FLB_TRACE             On)
   set(FLB_CHUNK_TRACE       On)
@@ -315,6 +321,12 @@ if (FLB_SYSTEM_LINUX)
   include(cmake/s390x.cmake)
 endif ()
 
+# Enable signed char support on Linux AARCH64 if specified
+if (FLB_LINUX_ON_AARCH64)
+  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsigned-char")
+  message(STATUS "Enabling signed char")
+endif()
+
 # Extract Git commit information for debug output.
 # Note that this is only set when cmake is run, the intent here is to use in CI for verification of releases so is acceptable.
 # For a better solution see https://jonathanhamberg.com/post/cmake-embedding-git-hash/ but this is simple and easy.

dockerfiles/Dockerfile

@@ -13,7 +13,7 @@
 # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/
 
 # Set this to the current release version: it gets done so as part of the release.
-ARG RELEASE_VERSION=3.2.0
+ARG RELEASE_VERSION=3.2.3
 
 # For multi-arch builds - assumption is running on an AMD64 host
 FROM multiarch/qemu-user-static:x86_64-arm AS qemu-arm32

fluent-bit-3.2.1.bb → fluent-bit-3.2.3.bb

@@ -16,7 +16,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93"
 SECTION = "net"
 
 PR = "r0"
-PV = "3.2.1"
+PV = "3.2.3"
 
 SRCREV = "v${PV}"
 SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1"

include/fluent-bit/flb_http_client.h

@@ -28,6 +28,8 @@
 #include <fluent-bit/flb_http_client_http1.h>
 #include <fluent-bit/flb_http_client_http2.h>
 
+#define HTTP_CLIENT_TEMPORARY_BUFFER_SIZE (1024 * 64)
+
 #define HTTP_CLIENT_SUCCESS         0
 #define HTTP_CLIENT_PROVIDER_ERROR -1
 
@@ -145,6 +147,71 @@ struct flb_http_debug {
     int (*cb_debug_request_payload);
 };
 
+/* To make opaque struct */
+struct flb_http_client;
+
+/*
+ * Tests callbacks
+ * ===============
+ */
+struct flb_test_http_response {
+    /*
+     * Response Test Mode
+     * ====================
+     * When the response test enable the test response mode, it needs to
+     * keep a reference of the context and other information:
+     *
+     * - rt_ctx : flb_http_client context
+     *
+     * - rt_status : HTTP response code
+     *
+     * - rt_in_callback: intermediary function to receive the results of
+     *                  the http response test function.
+     *
+     * - rt_data: opaque data type for rt_in_callback()
+     */
+
+    /* runtime library context */
+    void *rt_ctx;
+
+    /* HTTP status */
+    int rt_status;
+
+    /* optional response context */
+    void *response_ctx;
+
+    /*
+     * "response test callback": this function pointer is used by Fluent Bit
+     * http client testing mode to reference a test function that must retrieve the
+     * results of 'callback'. Consider this an intermediary function to
+     * transfer the results to the runtime test.
+     *
+     * This function is private and should not be set manually in the plugin
+     * code, it's set on src/flb_http_client.c .
+     */
+    void (*rt_resp_callback) (void *, int, void *, size_t, void *);
+
+    /*
+     * opaque data type passed by the runtime library to be used on
+     * rt_in_callback().
+     */
+    void *rt_data;
+
+    /*
+     * Callback
+     * =========
+     * "HTTP response callback": it references the plugin function that performs
+     * to validate HTTP response by HTTP client. This entry is mostly to
+     * expose the plugin local function.
+     */
+    int (*callback) (/* plugin that ingested the records */
+                     struct flb_http_client *,
+                     const void *,   /* incoming response data */
+                     size_t,         /* incoming response size */
+                     void **,        /* output buffer      */
+                     size_t *);      /* output buffer size */
+};
+
 /* Set a request type */
 struct flb_http_client {
     /* Upstream connection */
@@ -178,6 +245,10 @@ struct flb_http_client {
     /* Response */
     struct flb_http_client_response resp;
 
+    /* Tests */
+    int test_mode;
+    struct flb_test_http_response test_response;
+
     /* Reference to Callback context */
     void *cb_ctx;
 };
@@ -188,6 +259,7 @@ struct flb_http_client_ng {
     uint16_t                port;
     uint64_t                flags;
     int                     protocol_version;
+    cfl_sds_t               temporary_buffer;
 
     int                     releasable;
     void                   *user_data;
@@ -279,6 +351,13 @@ struct flb_http_client *flb_http_client(struct flb_connection *u_conn,
                                         const char *host, int port,
                                         const char *proxy, int flags);
 
+/* For fulfilling HTTP response testing (dummy client) */
+struct flb_http_client *flb_http_dummy_client(struct flb_connection *u_conn,
+                                              int method, const char *uri,
+                                              const char *body, size_t body_len,
+                                              const char *host, int port,
+                                              const char *proxy, int flags);
+
 int flb_http_add_header(struct flb_http_client *c,
                         const char *key, size_t key_len,
                         const char *val, size_t val_len);
@@ -294,6 +373,12 @@ int flb_http_set_keepalive(struct flb_http_client *c);
 int flb_http_set_content_encoding_gzip(struct flb_http_client *c);
 int flb_http_set_callback_context(struct flb_http_client *c,
                                   struct flb_callback *cb_ctx);
+int flb_http_set_response_test(struct flb_http_client *c, char *test_name,
+                               const void *data, size_t len,
+                               int status,
+                               void (*resp_callback) (void *, int, void *, size_t, void *),
+                               void *resp_callback_data);
+int flb_http_push_response(struct flb_http_client *c, const void *data, size_t len);
 
 int flb_http_get_response_data(struct flb_http_client *c, size_t bytes_consumed);
 int flb_http_do_request(struct flb_http_client *c, size_t *bytes);

include/fluent-bit/flb_http_client_http2.h

@@ -22,6 +22,7 @@
 
 #include <fluent-bit/flb_http_common.h>
 #include <nghttp2/nghttp2.h>
+#include <nghttp2/nghttp2.h>
 
 struct flb_http_client_session;
 

include/fluent-bit/flb_http_common.h

@@ -88,6 +88,7 @@ struct flb_http_server_session;
 
 struct flb_http_request {
     int                               protocol_version;
+    cfl_sds_t                         authority;
     int                               method;
     cfl_sds_t                         path;
     cfl_sds_t                         host;

include/fluent-bit/flb_lib.h

@@ -68,6 +68,9 @@ FLB_EXPORT int flb_output_set_test(flb_ctx_t *ctx, int ffd, char *test_name,
                                    void *test_ctx);
 FLB_EXPORT int flb_output_set_callback(flb_ctx_t *ctx, int ffd, char *name,
                                        void (*cb)(char *, void *, void *));
+FLB_EXPORT int flb_output_set_http_test(flb_ctx_t *ctx, int ffd, char *test_name,
+                                        void (*out_response) (void *, int, int, void *, size_t, void *),
+                                        void *out_callback_data);
 
 FLB_EXPORT int flb_filter_set(flb_ctx_t *ctx, int ffd, ...);
 FLB_EXPORT int flb_service_set(flb_ctx_t *ctx, ...);
@@ -84,6 +87,9 @@ FLB_EXPORT int flb_loop(flb_ctx_t *ctx);
 FLB_EXPORT int flb_lib_push(flb_ctx_t *ctx, int ffd, const void *data, size_t len);
 FLB_EXPORT int flb_lib_config_file(flb_ctx_t *ctx, const char *path);
 
+/* Emulate ingestions of HTTP responses for output plugins */
+FLB_EXPORT int flb_lib_response(flb_ctx_t *ctx, int ffd, int status, const void *data, size_t len);
+
 /* library context handling */
 FLB_EXPORT void flb_context_set(flb_ctx_t *ctx);
 FLB_EXPORT flb_ctx_t *flb_context_get();

include/fluent-bit/flb_output.h

@@ -168,6 +168,66 @@ struct flb_test_out_formatter {
                      size_t *);      /* output buffer size */
 };
 
+struct flb_test_out_response {
+    /*
+     * Runtime Library Mode
+     * ====================
+     * When the runtime library enable the test formatter mode, it needs to
+     * keep a reference of the context and other information:
+     *
+     * - rt_ctx : context created by flb_create()
+     *
+     * - rt_ffd : this plugin assigned 'integer' created by flb_output()
+     *
+     * - rt_step_calback: intermediary function to receive the results of
+     *                    the formatter plugin test function.
+     *
+     * - rt_data: opaque data type for rt_step_callback()
+     */
+
+    /* runtime library context */
+    void *rt_ctx;
+
+    /* runtime library: assigned plugin integer */
+    int rt_ffd;
+
+    /*
+     * "runtime step callback": this function pointer is used by Fluent Bit
+     * library mode to reference a test function that must retrieve the
+     * results of 'callback'. Consider this an intermediary function to
+     * transfer the results to the runtime test.
+     *
+     * This function is private and should not be set manually in the plugin
+     * code, it's set on src/flb_lib.c .
+     */
+    void (*rt_out_response) (void *, int, int, void *, size_t, void *);
+
+    /*
+     * opaque data type passed by the runtime library to be used on
+     * rt_step_test().
+     */
+    void *rt_data;
+
+    /* optional context for flush callback */
+    void *flush_ctx;
+
+    /*
+     * Callback
+     * =========
+     * "Formatter callback": it references the plugin function that performs
+     * data formatting (msgpack -> local data). This entry is mostly to
+     * expose the plugin local function.
+     */
+    int (*callback) (/* Fluent Bit context */
+                     struct flb_config *,
+                     void *,         /* plugin instance context */
+                     int status,     /* HTTP status code */
+                     const void *,   /* respond msgpack data */
+                     size_t,         /* respond msgpack size */
+                     void **,        /* output buffer      */
+                     size_t *);      /* output buffer size */
+};
+
 struct flb_output_plugin {
     /*
      * a 'mask' to define what kind of data the plugin can manage:
@@ -241,6 +301,7 @@ struct flb_output_plugin {
 
     /* Tests */
     struct flb_test_out_formatter test_formatter;
+    struct flb_test_out_response test_response;
 
     /* Link to global list from flb_config->outputs */
     struct mk_list _head;
@@ -391,6 +452,7 @@ struct flb_output_instance {
 
     /* Tests */
     struct flb_test_out_formatter test_formatter;
+    struct flb_test_out_response test_response;
 
     /*
      * Buffer counter: it counts the total of disk space (filesystem) used by buffers