Merge branch 'mainline' into fips-image

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,29 @@
+<!--
+Please make sure you've read and understood our contributing guidelines;
+https://github.com/aws/aws-for-fluent-bit/blob/mainline/CONTRIBUTING.md
+
+Please provide the following information:
+-->
+
+### Summary
+<!-- What does this pull request do? -->
+
 *Issue #, if available:*
 
-*Description of changes:*
+### Testing
+<!-- How was this tested?
+See https://github.com/aws/aws-for-fluent-bit?tab=readme-ov-file#local-testing
+for instructions on how to run integ tests locally.
+-->
+
+New tests cover the changes: <!-- yes|no -->
+
+### Description for the changelog
+<!--
+Write a short (one line) summary that describes the changes in this
+pull request for inclusion in the changelog.
+-->
 
+### Licensing
 
-By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
+By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

.gitignore

@@ -1,3 +1,3 @@
 bin
-integ/out
+integ/out/
 .venv

AWS_FLB_CHERRY_PICKS

@@ -1,145 +0,0 @@
-# Revert datadog fix PR to resolve segfault
-https://github.com/matthewfala/fluent-bit.git ecs-datadog-sequential-revert 98313ebf206eec4a4e5375b352fc36849b762323
-
-# Support Opensearch Serverless data ingestion
-https://github.com/matthewfala/fluent-bit.git aoss-1.9 1633c49aadad55bac483c5e55772de0e6c29704a
-https://github.com/matthewfala/fluent-bit.git aoss-1.9 e1301bc52e209b6c6de8602bff12e0a98aefa0c0
-https://github.com/matthewfala/fluent-bit.git aoss-1.9 1917a7760d1bd400ac987620661b48e1bbc001ae
-
-# Kinesis time format
-https://github.com/Claych/fluent-bit.git clay-aws_strftime_precision-1.9 dfeff9de13ba7f1bbebe08fef24ec993dee7e392
-https://github.com/Claych/fluent-bit.git clay-aws_strftime_precision-1.9 66e85a11590f0045294a3d1b104baf73569a1dd0
-https://github.com/Claych/fluent-bit.git clay-aws_strftime_precision-1.9 e225ff7374fbc5e290b0090652158e3d2db5d8a6
-
-# S3 log_key warn fix
-https://github.com/PettitWesley/fluent-bit.git s3-log-key-warn-1_9 308b73558fb2d3dafb3b80feb0c68dc9b2c18186
-
-# ECS Filter
-https://github.com/PettitWesley/fluent-bit.git ecs-empty-metadata-fix-one-commit 24934c938e8f63700edf1230a09e4483ac5df6a3
-
-# Add back Datadog fixes that do not trigger segfault
-https://github.com/matthewfala/fluent-bit.git ecs-datadog-sequential-revert acc01a4bf4ce656023d82943bec2683b5b7755e5
-https://github.com/matthewfala/fluent-bit.git ecs-datadog-sequential-revert ac30b7c876a95d44c33a406b2b58ce08b978587e
-
-# Datadog Partial Fix Patch
-https://github.com/matthewfala/fluent-bit.git datadog-ecs-patch 3c1ad69ada5bb6f2e448c6f39a1a0ea6a6f4ff17
-
-# Resolve cloudwatch_logs duplicate tag match SIGSEGV issue
-https://github.com/matthewfala/fluent-bit.git sync-scheduler-fix-1.9 2614c46af3a051b2758bef57f01f6b10a8e73b62
-
-# resolve user agent wrong type
-https://github.com/PettitWesley/fluent-bit.git user-agent-type-fix 0642f42e8097c159d7364f1ff97c0196484815ce
-https://github.com/PettitWesley/fluent-bit.git user-agent-type-fix ed21492a94e8ca156897afc32c73bcdc37bc6b0f
-https://github.com/PettitWesley/fluent-bit.git user-agent-type-fix 7c16af941bed91da9558e4e6b8cea98d3ef1fa0d
-
-# Resolve keepalive and priority scheduler issue
-https://github.com/PettitWesley/fluent-bit.git sync-io-keepalive-fix 7b4550486b7e8e02b773894d31a08b0cb18154e3
-
-https://github.com/PettitWesley/fluent-bit.git mk_event_inject_conn_fix 0f5efc921950feeee99d9251cb330b27d3639863
-
-https://github.com/PettitWesley/fluent-bit.git mk_event_add_corruption-one-commit 33651cca41e9f84ce8930a9bca9d3d7319e50fbd
-
-# Useful debug messages for input events
-# input chunk append message with input name context and number of records
-https://github.com/PettitWesley/fluent-bit.git chunk-append-context b671ed38e7ae87ab955083e2324ef1972298c5d7
-
-# in_tail file name context for inotify events
-https://github.com/PettitWesley/fluent-bit.git tail-modify-debug-context-immutable-cherry-pick ab11d1d7438f5254d04eba8dfc2f10b78cc2244d
-https://github.com/PettitWesley/fluent-bit.git tail-modify-debug-context-immutable-cherry-pick 1f3bdeec49ac42b04b921303610ea0f53110ab2c
-
-# S3 tag corruption fix
-https://github.com/PettitWesley/fluent-bit.git s3-str-fixes 38303131e049265277881c0d79935ad31fdd3e13
-
-# User friend message when storage.total_limit_size causes fs chunk deletion, before there was no indication
-https://github.com/fluent/fluent-bit.git master b725d6b8b289fccde4e9b31d3f3ac61f13711ef9
-# use total_chunks_up in max_chunks_up memory overlimit warn message
-https://github.com/fluent/fluent-bit.git master 9c72f3ac6510b701277936897cd9701ffce3646e
-
-# CloudWatch Logs options for connecting to CWL test destinations: tls verify and port
-https://github.com/matthewfala/fluent-bit.git immutable-cwl-net-options 5d9692f00b5295728bf0340d332896a7cc450a7e
-
-# Go exit fix
-https://github.com/PettitWesley/fluent-bit.git go-exit-fix-1_9-one-commit ce5739c20b972320dc485587d56c8b6b21f61934
-# fix build warning from original go fix
-https://github.com/PettitWesley/fluent-bit.git fix-proxy-go-destroy 79e4e10f31b7468496d4dddb784b502b3ba9e353
-
-# sds printf off by 1 fix: https://github.com/fluent/fluent-bit/issues/7143
-# from PR: https://github.com/fluent/fluent-bit/pull/7148/commits
-https://github.com/PettitWesley/fluent-bit.git sds-off-by-1-1_9 e7ba91a6c05d884cc6745d8e49faeb1a92909679
-https://github.com/PettitWesley/fluent-bit.git sds-off-by-1-1_9 6c9e49a627931bd1bdbd8d965a64bfd5c325e01d
-https://github.com/PettitWesley/fluent-bit.git sds-off-by-1-1_9 f45b3027dfd0ebac20e35df16bed14020718b780
-
-# cw mem leak fix (leak is only a few bytes no matter the runtime/throughput) https://github.com/fluent/fluent-bit/pull/7158/commits
-https://github.com/PettitWesley/fluent-bit.git cw-stream-free-fix 8e7809ee9f4e7837a5fff75842a47ca5fd42b526
-
-# Messagepack Fix https://github.com/fluent/fluent-bit/commit/c0fc0374c54ae5967f12b5ac34ce89a0ca285210
-https://github.com/fluent/fluent-bit.git 1.9 c0fc0374c54ae5967f12b5ac34ce89a0ca285210
-
-# STS response parsing improvement/fix
-https://github.com/PettitWesley/fluent-bit.git sts-response-parse-fix b1186b92b53466a240b1f16008995dc85afed892
-
-# upstream config map fix https://github.com/fluent/fluent-bit/pull/6874
-https://github.com/fluent/fluent-bit.git 1.9 81cdf7eced4e420043277237fba092157b17ffd9
-
-# upstream engine retry clean up fix https://github.com/fluent/fluent-bit/pull/6862
-https://github.com/fluent/fluent-bit.git 1.9 712e5fbe10bee44269d5dfed214c4e087ea1ec2a
-
-# quick fix for S3 key $INDEX bug https://github.com/aws/aws-for-fluent-bit/issues/653
-https://github.com/PettitWesley/fluent-bit.git index_s3_key_format_quick_fix 5e48218670681aef152aeedcf90a4593ac623470
-
-# lib upgrades: chunkio to 1.4.0 and monkey to commit 13a4ccd3
-https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades d56634674725aee5101fd17845730bbb66318928
-https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades e34af51fee5b5b11f0d239c80308a6bb80ab5f13
-# libbacktrace to 8602fda, cfl to v0.2.3, onigmo to 2bfee1eaf526ec2309822243a976cc792d99fbc3, lib: upgrade to 4bd9260
-https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades bd7ad8943e79d8515ceed8ef005a7440ef78ecf6
-
-# several upstream aws_util memory fixes (see commit message)
-https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades 97047e018cfcb1f79daecd7d3ccfbff21948e246
-
-# upstream 2.x in_exec bug fixes
-# resolves https://github.com/aws/aws-for-fluent-bit/issues/661
-https://github.com/fluent/fluent-bit.git master 6ed4aaabd063b8fdf0c034729e45429da87dc142
-https://github.com/fluent/fluent-bit.git master 62431ad0aede70d9748c372ebc7ac9a9917f9c9d
-
-# Tail memory fix picked from here: https://github.com/fluent/fluent-bit/commit/ed758a5eb85967cc66ca8dff269e7454b2394c3a
-https://github.com/PettitWesley/fluent-bit.git upstream-tail-stat-fix 800bb813a00f14a5f457cc69a89f8fb0f715e8c4
-
-# in_http: fix memory initialization and enable it on windows https://github.com/fluent/fluent-bit/issues/7008
-https://github.com/fluent/fluent-bit.git master 7a882df735b28002983770f554b365dc63c0be7e
-
-# record_accessor/rewrite_tag fix to allow single character rules: https://github.com/fluent/fluent-bit/issues/7330
-https://github.com/PettitWesley/fluent-bit.git 2_31_12_lib_upgrades 4c5c8ab56075b7ce63023f8c5c0c963200027a67
-
-# output thread, fix memory initialization: https://github.com/fluent/fluent-bit/pull/7303
-https://github.com/fluent/fluent-bit.git master 9a08168a8ab293fc8054180ee04e1176469df88b
-
-# filter_modify: fix memory clean up: https://github.com/fluent/fluent-bit/issues/7368
-https://github.com/PettitWesley/fluent-bit.git filter-modify-fix-aws-distro 1a72de13ad6cfd5a176e5d8712064a38a7d097f2
-
-# AWS core code self-review issue fixes
-# upstream version of this: https://github.com/fluent/fluent-bit/pull/7512/files
-# use calloc in all credential code to prevent freeing of garbage pointers 
-https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review eb48b79b34f91d36d28434390b976e882d553681
-# fix brittle XML parsing
-https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review e6401ad3811b42dee0b7f92aba726cca4bee74ec
-# add pthread_mutex + trylock to protect cred providers
-https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review a1d7469da62a4b3ca869b10732f41562d668cfce
-# signv4: always use calloc
-https://github.com/PettitWesley/fluent-bit.git aws-distro-crypto-self-review 3381c388ee956e7d0e7c0d5fc44683da75095a6a
-
-# Cloudwatch_logs sequence token deprecation
-https://github.com/matthewfala/fluent-bit.git immutable-sequence-token-deprecation 8ee560e388bbbf850069c81bbca06275f330baeb
-
-# prometheus: sigv4 aws-for-fluent-bit
-https://github.com/matthewfala/fluent-bit.git immutable-2.32.0-prometheus-sigv4 ca93bd1d43ebedeb8e81b46b800ea229fde66fa5
-
-# multiline: remove incorrect flush
-https://github.com/matthewfala/fluent-bit.git immutable-multiline-incorrect-flush 6431a4e584d52170dbe873d93ba532659921740a
-https://github.com/matthewfala/fluent-bit.git immutable-multiline-incorrect-flush 35f23875ca356ea30e9aac19854b810cf8ecad8f
-
-# core: network event drop shutdown fix
-https://github.com/matthewfala/fluent-bit.git 2.32.0-premature-connection-destruction 9e2e5d1bffca92bbcc5001fcfc34c1d9ae2716db
-https://github.com/matthewfala/fluent-bit.git 2.32.0-premature-connection-destruction b2e8ff1ae738c1db7bf50942ef619609436ffe02
-
-# throttle: print_status configuration issue resolution
-https://github.com/matthewfala/fluent-bit.git throttle-filter-print-status-fix 7b05b7ebfe55261ed12d5006c8b682572b6abf4c

AWS_FOR_FLUENT_BIT_STABLE_VERSION

@@ -1 +1 @@
-2.32.2.20240820
+2.32.2.20241008

AWS_FOR_FLUENT_BIT_VERSION

@@ -1 +1 @@
-2.32.2.20240820
+2.32.2.20241008

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+### 2.32.2.20241008 Linux re-build
+
+*This release has the same Fluent Bit contents as 2.32.2. It is a linux-only re-build to mitigate a code bug in the new change-management system in 2.32.2.20241003. There are no windows images for this release.* 
+
+### 2.32.2.20241003 Linux re-build
+
+*This release has the same Fluent Bit contents as 2.32.2. It is a linux-only re-build to switch to a new change-management system and merge in recent patches in dependencies installed in the image. There are no windows images for this release.* 
+* Amazon Linux Base: [2.0.20240916.0](https://docs.aws.amazon.com/AL2/latest/relnotes/relnotes-20240916.html)
+
+
 ### 2.32.2.20240820 Linux re-build
 
 *This release has the same Fluent Bit contents as 2.32.2, and is simply a linux-only re-build for recent patches in dependencies installed in the image. There are no windows images for this release.* 

Makefile

@@ -13,8 +13,12 @@
 
 all: release
 
-# Improve build speeds during development by removing the --no-cache flag
-export DOCKER_BUILD_FLAGS=--no-cache
+# Execute set-cache to turn docker cache back on for faster development.
+DOCKER_BUILD_FLAGS := "--no-cache"
+
+.PHONY: dev
+dev: DOCKER_BUILD_FLAGS =
+dev: release
 
 .PHONY: release
 release: build build-init build-fips linux-plugins linux-plugins-fips
@@ -154,40 +158,43 @@ kinesis-dev:
 	$(DOCKER_BUILD_FLAGS) -t aws-fluent-bit-plugins:latest -f ./scripts/dockerfiles/Dockerfile.plugins .
 	docker build -t amazon/aws-for-fluent-bit:latest -f ./scripts/dockerfiles/Dockerfile .
 
+integ/out:
+	mkdir -p integ/out
+
 .PHONY: integ-cloudwatch
-integ-cloudwatch: release
+integ-cloudwatch: integ/out release
 	./integ/integ.sh cloudwatch
 
 .PHONY: integ-cloudwatch-dev
-integ-cloudwatch-dev: cloudwatch-dev
+integ-cloudwatch-dev: integ/out cloudwatch-dev
 	./integ/integ.sh cloudwatch
 
 .PHONY: integ-clean-cloudwatch
-integ-clean-cloudwatch:
+integ-clean-cloudwatch: integ/out
 	./integ/integ.sh clean-cloudwatch
 
 .PHONY: integ-kinesis
-integ-kinesis: release
+integ-kinesis: integ/out release
 	./integ/integ.sh kinesis
 
 .PHONY: integ-kinesis-dev
-integ-kinesis-dev: kinesis-dev
+integ-kinesis-dev: integ/out kinesis-dev
 	./integ/integ.sh kinesis
 
 .PHONY: integ-firehose
-integ-firehose: release
+integ-firehose: integ/out release
 	./integ/integ.sh firehose
 
 .PHONY: integ-firehose-dev
-integ-firehose-dev: firehose-dev
+integ-firehose-dev: integ/out firehose-dev
 	./integ/integ.sh firehose
 
 .PHONY: integ-clean-s3
-integ-clean-s3:
+integ-clean-s3: integ/out
 	./integ/integ.sh clean-s3
 
 .PHONY: integ-dev
-integ-dev: release
+integ-dev: integ/out dev
 	./integ/integ.sh kinesis
 	./integ/integ.sh kinesis_streams
 	./integ/integ.sh firehose
@@ -196,7 +203,7 @@ integ-dev: release
 	./integ/integ.sh cloudwatch_logs
 
 .PHONY: integ
-integ:
+integ: integ/out
 	./integ/integ.sh cicd
 
 .PHONY: delete-resources
@@ -206,6 +213,7 @@ delete-resources:
 .PHONY: clean
 clean:
 	rm -rf ./build
+	rm -rf ./integ/out
 	docker image remove -f aws-fluent-bit-plugins:latest
 	docker image remove -f amazon/aws-fluent-bit-plugins:fips-latest
 

README.md

@@ -354,16 +354,32 @@ For more details about running Fluent Bit Windows containers in Amazon ECS, plea
 
 ### Development
 
-#### Local testing
+#### Local integ testing
 
-Use `make release` to build the image.
+Use `make dev` to build the image.
 
-To run the integration tests, run `make integ-dev`. The `make integ-dev` command will run the integration tests for all of our plugins-
+The `make integ-dev` command will run the integration tests for all of our plugins-
 kinesis streams, kinesis firehose, and cloudwatch.
 
-The integ tests require the following env vars to be set:
-* `CW_INTEG_VALIDATOR_IMAGE`: Build the [integ/validate_cloudwatch/](integ/validate_cloudwatch/) folder with `docker build` and set the resulting image as the value of this env var.
-* `S3_INTEG_VALIDATOR_IMAGE`: Build the [integ/s3/](integ/s3/) folder with `docker build` and set the resulting image as the value of this env var.
+Note that these steps rely on creating Cfn stacks in an AWS account in region us-west-2,
+so AWS credentials must be setup before they are run.
+
+Instructions:
+1. Setup AWS access via EC2 instance role or AWS_* env vars
+2. Install dependent packages: `docker awscli`
+3. Install docker-compose:
+```
+sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+```
+4. Build validator images:
+```
+pushd integ/validate_cloudwatch && docker build -t flbcwinteg . && popd
+pushd integ/s3 && docker build -t flbs3integ . && popd
+export CW_INTEG_VALIDATOR_IMAGE="flbcwinteg"
+export S3_INTEG_VALIDATOR_IMAGE="flbs3integ"
+```
+5. Run `make integ-dev`
 
 To run integration tests separately, execute `make integ-cloudwatch` or `make integ-kinesis` or `make integ-firehose`.
 

integ/integ.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+
+set -ex
+
 export AWS_REGION="us-west-2"
 export PROJECT_ROOT="$(pwd)"
 export VOLUME_MOUNT_CONTAINER="/out"
@@ -9,8 +12,14 @@ if [ "$ARCHITECTURE" = "aarch64" ]; then
     ARCHITECTURE="arm64"
 fi
 
-export CW_INTEG_VALIDATOR_IMAGE="${CW_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
-export S3_INTEG_VALIDATOR_IMAGE="${S3_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
+# If we're testing locally, then these are set to local images rather than pulling
+# from ECR. See https://github.com/aws/aws-for-fluent-bit?tab=readme-ov-file#local-testing
+if [ -z "$CW_INTEG_VALIDATOR_IMAGE" ]; then
+	export CW_INTEG_VALIDATOR_IMAGE="${CW_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
+fi
+if [ -z "$S3_INTEG_VALIDATOR_IMAGE" ]; then
+	export S3_INTEG_VALIDATOR_IMAGE="${S3_INTEG_VALIDATOR_IMAGE_BASE}:${ARCHITECTURE}"
+fi
 
 test_cloudwatch() {
 	export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}"
@@ -287,7 +296,7 @@ if [ "${1}" = "cicd" ]; then
 	export TEST_FILE="kinesis-test"
 	export EXPECTED_EVENTS_LEN="1000"
 	clean_s3 && test_kinesis
-	
+
 	# golang firehose plugin
 	export S3_PREFIX="firehose-test"
 	export TEST_FILE="firehose-test"
@@ -315,4 +324,4 @@ fi
 
 if [ "${1}" = "delete" ]; then
 	source ./integ/resources/delete_test_resources.sh
-fi
+fi

integ/resources/create_test_resources.sh

@@ -8,4 +8,4 @@ ARCHITECTURE=$(uname -m | tr '_' '-')
 if [ "$ARCHITECTURE" = "aarch64" ]; then
     ARCHITECTURE="arm64"
 fi
-aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region us-west-2 --capabilities CAPABILITY_NAMED_IAM
+aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit-${ARCHITECTURE} --region "$AWS_REGION" --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset

integ/resources/setup_test_environment.sh

@@ -7,7 +7,7 @@ ARCHITECTURE=$(uname -m | tr '_' '-')
 if [ "$ARCHITECTURE" = "aarch64" ]; then
     ARCHITECTURE="arm64"
 fi
-stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue')
+stackOutputs=$(aws cloudformation describe-stacks --region "$AWS_REGION" --stack-name integ-test-fluent-bit-${ARCHITECTURE} --output text --query 'Stacks[0].Outputs[*].OutputValue')
 read -r -a outputArray <<< "$stackOutputs"
 export FIREHOSE_STREAM="${outputArray[0]}"
 export KINESIS_STREAM="${outputArray[1]}"

integ/s3/go.mod

@@ -4,6 +4,6 @@ go 1.17
 
 require github.com/aws/aws-sdk-go v1.44.232
 
-require golang.org/x/net v0.7.0 // indirect
+require golang.org/x/net v0.23.0 // indirect
 
 require github.com/jmespath/go-jmespath v0.4.0 // indirect