diff --git a/AWS_FLB_CHERRY_PICKS b/AWS_FLB_CHERRY_PICKS index ad85b7e52..11c8df271 100644 --- a/AWS_FLB_CHERRY_PICKS +++ b/AWS_FLB_CHERRY_PICKS @@ -42,6 +42,7 @@ https://github.com/PettitWesley/fluent-bit.git mk_event_add_corruption-one-commi # Useful debug messages for input events # input chunk append message with input name context and number of records https://github.com/PettitWesley/fluent-bit.git chunk-append-context b671ed38e7ae87ab955083e2324ef1972298c5d7 + # in_tail file name context for inotify events https://github.com/PettitWesley/fluent-bit.git tail-modify-debug-context-immutable-cherry-pick ab11d1d7438f5254d04eba8dfc2f10b78cc2244d https://github.com/PettitWesley/fluent-bit.git tail-modify-debug-context-immutable-cherry-pick 1f3bdeec49ac42b04b921303610ea0f53110ab2c @@ -83,15 +84,6 @@ https://github.com/fluent/fluent-bit.git 1.9 81cdf7eced4e420043277237fba092157b1 # upstream engine retry clean up fix https://github.com/fluent/fluent-bit/pull/6862 https://github.com/fluent/fluent-bit.git 1.9 712e5fbe10bee44269d5dfed214c4e087ea1ec2a -# TCP input: user friendly message for skipping logs https://github.com/fluent/fluent-bit/pull/6061 -https://github.com/fluent/fluent-bit.git 1.9 c5d57ccf4f85af77e2a2bbb1121df95887e2962f - -# ElasticSearch: fix buffer bulk over run https://github.com/fluent/fluent-bit/pull/5770 -https://github.com/fluent/fluent-bit.git 1.9 d3a0697c3c4a876250cc6c2c4ff44cd77908ea3c - -# windows: fix time zone offset https://github.com/fluent/fluent-bit/pull/6368 -https://github.com/fluent/fluent-bit.git 1.9 cae9a8f9cb6eab7449f14d914f9ba1d8d4e93cf0 - # quick fix for S3 key $INDEX bug https://github.com/aws/aws-for-fluent-bit/issues/653 https://github.com/PettitWesley/fluent-bit.git index_s3_key_format_quick_fix 5e48218670681aef152aeedcf90a4593ac623470 @@ -127,7 +119,7 @@ https://github.com/PettitWesley/fluent-bit.git filter-modify-fix-aws-distro 1a72 # AWS core code self-review issue fixes # upstream version of this: https://github.com/fluent/fluent-bit/pull/7512/files # use calloc in all credential code to prevent freeing of garbage pointers -https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review eb48b79b34f91d36d28434390b976e882d553681 +https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review eb48b79b34f91d36d28434390b976e882d553681 # fix brittle XML parsing https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review e6401ad3811b42dee0b7f92aba726cca4bee74ec # add pthread_mutex + trylock to protect cred providers @@ -135,3 +127,19 @@ https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review a1d # signv4: always use calloc https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review 3381c388ee956e7d0e7c0d5fc44683da75095a6a +# Cloudwatch_logs sequence token deprecation +https://github.com/matthewfala/fluent-bit.git immutable-sequence-token-deprecation 8ee560e388bbbf850069c81bbca06275f330baeb + +# prometheus: sigv4 aws-for-fluent-bit +https://github.com/matthewfala/fluent-bit.git immutable-2.32.0-prometheus-sigv4 ca93bd1d43ebedeb8e81b46b800ea229fde66fa5 + +# multiline: remove incorrect flush +https://github.com/matthewfala/fluent-bit.git immutable-multiline-incorrect-flush 6431a4e584d52170dbe873d93ba532659921740a +https://github.com/matthewfala/fluent-bit.git immutable-multiline-incorrect-flush 35f23875ca356ea30e9aac19854b810cf8ecad8f + +# core: network event drop shutdown fix +https://github.com/matthewfala/fluent-bit.git 2.32.0-premature-connection-destruction 9e2e5d1bffca92bbcc5001fcfc34c1d9ae2716db +https://github.com/matthewfala/fluent-bit.git 2.32.0-premature-connection-destruction b2e8ff1ae738c1db7bf50942ef619609436ffe02 + +# throttle: print_status configuration issue resolution +https://github.com/matthewfala/fluent-bit.git throttle-filter-print-status-fix 7b05b7ebfe55261ed12d5006c8b682572b6abf4c diff --git a/AWS_FOR_FLUENT_BIT_VERSION b/AWS_FOR_FLUENT_BIT_VERSION index 86ff3a0cc..7cca401c7 100644 --- a/AWS_FOR_FLUENT_BIT_VERSION +++ b/AWS_FOR_FLUENT_BIT_VERSION @@ -1 +1 @@ -2.31.12.20231011 \ No newline at end of file +2.32.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ac57470d..d5f2f89ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +### 2.32.0 +This release includes: +* Fluent Bit [1.9.10](https://github.com/fluent/fluent-bit/tree/v1.9.10) +* Amazon CloudWatch Logs for Fluent Bit 1.9.4 +* Amazon Kinesis Streams for Fluent Bit 1.10.2 +* Amazon Kinesis Firehose for Fluent Bit 1.7.2 + +Compared to `2.31.12` this release adds: +* Feature - Customers can send metrics to Amazon Managed Prometheus via added sigv4 authentication on `prometheus_remote_write`. Refer to [amazon-ecs-firelens-examples](https://github.com/aws-samples/amazon-ecs-firelens-examples/blob/mainline/examples/fluent-bit/amazon-managed-service-for-prometheus/README.md) for information on how to export metrics to AMP on ECS [aws-for-fluent-bit:256](https://github.com/aws/aws-for-fluent-bit/issues/256) +* Feature - Support multiline parsers with the init tag. Multiline parsers can be specified same as a standard parser file [aws-for-fluent-bit:537](https://github.com/aws/aws-for-fluent-bit/issues/537) +* Enhancement - Customers can route logs to CloudWatch Logs at higher throughputs by increasing number of output workers as `cloudwatch_logs` output plugin removed sequence tokens from API requests [aws-for-fluent-bit:526](https://github.com/aws/aws-for-fluent-bit/issues/526) +* Fix - Fix multiline input behavior when multiple streams are parsed (stderr, stdout) together. Multiline logs are no longer terminated when streams are switched between [fluent-bit:7469](https://github.com/fluent/fluent-bit/pull/7469) +* Fix - Fix networking edgecase causing data loss and OOM related issues via net timeout event injection resolution [fluent-bit:7728](https://github.com/fluent/fluent-bit/pull/7728/files) +* Fix - Allow reduced filter_throttle log output with `print_status` `false` configuration option fix [fluent-bit:7469](https://github.com/fluent/fluent-bit/pull/6500/files) + ### 2.31.12.20231011 Linux re-build *This release has the same Fluent Bit contents as 2.31.12, and is simply a linux-only re-build for recent patches in dependencies installed in the image. There are no windows images for this release.* diff --git a/Makefile b/Makefile index c205dee5e..95fab8ad8 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ all: release export DOCKER_BUILD_FLAGS=--no-cache .PHONY: release -release: build build-init +release: build build-init linux-plugins docker system prune -f docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:main-release -f ./scripts/dockerfiles/Dockerfile.main-release . docker tag amazon/aws-for-fluent-bit:main-release amazon/aws-for-fluent-bit:latest @@ -28,7 +28,7 @@ release: build build-init debug: main-debug init-debug .PHONY: build -build: linux-plugins +build: docker system prune -f docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:build -f ./scripts/dockerfiles/Dockerfile.build . @@ -118,7 +118,7 @@ init-debug-s3: main-debug-base build-init docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:init-debug-s3 -f ./scripts/dockerfiles/Dockerfile.init-debug-s3 . .PHONY: main-debug-base -main-debug-base: build +main-debug-base: build linux-plugins docker build $(DOCKER_BUILD_FLAGS) -t amazon/aws-for-fluent-bit:main-debug-base -f ./scripts/dockerfiles/Dockerfile.main-debug-base . .PHONY: validate-version-file-format diff --git a/linux.version b/linux.version index 553c3b910..907e8e58f 100644 --- a/linux.version +++ b/linux.version @@ -1,6 +1,6 @@ { "linux": { - "version": "2.31.12.20231011", + "version": "2.32.0", "latest": "true", "build": "1", "fluent-bit": "1.9.10", diff --git a/scripts/dockerfiles/Dockerfile.build b/scripts/dockerfiles/Dockerfile.build index 51bebd84e..2d827bd73 100644 --- a/scripts/dockerfiles/Dockerfile.build +++ b/scripts/dockerfiles/Dockerfile.build @@ -88,14 +88,15 @@ COPY AWS_FLB_CHERRY_PICKS \ RUN git config --global user.email "aws-firelens@amazon.com" \ && git config --global user.name "FireLens Team" -RUN AWS_FLB_CHERRY_PICKS_COUNT=`cat /AWS_FLB_CHERRY_PICKS | sed '/^#/d' | sed '/^\s*$/d' | wc -l | awk '{ print $1 }'`; \ +RUN AWS_FLB_CHERRY_PICKS_COUNT=`awk '{print $0 }' /AWS_FLB_CHERRY_PICKS | sed '/^#/d' | sed '/^\s*$/d' | wc -l | awk '{ print $1 }'`; echo $AWS_FLB_CHERRY_PICKS_COUNT; \ if [ $AWS_FLB_CHERRY_PICKS_COUNT -gt 0 ]; \ then \ cat /AWS_FLB_CHERRY_PICKS | sed '/^#/d' \ - | xargs -l bash -c 'git fetch $0 $1 && git cherry-pick $2'; \ + | xargs -l bash -c 'git fetch $0 $1 && git cherry-pick $2 || exit 255' && \ \ - echo "Cherry Pick Patch Summary:"; \ + (echo "Cherry Pick Patch Summary:"; \ + echo -n "Base "; \ git log --oneline \ -$((AWS_FLB_CHERRY_PICKS_COUNT+1)) \ - | tac | awk '{ print "Commit",NR-1,"--",$0 }'; sleep 2; \ + | tac | awk '{ print "Commit",NR-1,"--",$0 }'; sleep 2;)\ fi diff --git a/windows.versions b/windows.versions index a80dc86e0..fd2ece42a 100644 --- a/windows.versions +++ b/windows.versions @@ -1,7 +1,7 @@ { "windows": [ { - "version": "2.31.12", + "version": "2.32.0", "build": "1", "fluent-bit": "1.9.10", "kinesis-plugin": "v1.10.2", @@ -10,6 +10,18 @@ "openssl": "3.0.7", "flexBison": "2.5.22", "latest": true, + "stable": false + }, + { + "version": "2.31.12", + "build": "1", + "fluent-bit": "1.9.10", + "kinesis-plugin": "v1.10.2", + "firehose-plugin": "v1.7.2", + "cloudwatch-plugin": "v1.9.4", + "openssl": "3.0.7", + "flexBison": "2.5.22", + "latest": false, "stable": true }, {