Merge branch 'aws:mainline' into fips-image

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,10 +12,12 @@ Please provide the following information:
 
 ### Testing
 <!-- How was this tested?
-See https://github.com/aws/aws-for-fluent-bit?tab=readme-ov-file#local-testing
+See https://github.com/aws/aws-for-fluent-bit?tab=readme-ov-file#local-integ-testing
 for instructions on how to run integ tests locally.
 -->
 
+`make debug` succeeded: <!-- yes|no -->
+Integ tests succeeded: <!-- yes|no -->
 New tests cover the changes: <!-- yes|no -->
 
 ### Description for the changelog

AWS_FOR_FLUENT_BIT_VERSION

@@ -1 +1 @@
-2.32.2.20241008
+2.32.4

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+### 2.32.4
+This release includes:
+* Fluent Bit [1.9.10](https://github.com/fluent/fluent-bit/tree/v1.9.10)
+* Amazon CloudWatch Logs for Fluent Bit 1.9.4
+* Amazon Kinesis Streams for Fluent Bit 1.10.2
+* Amazon Kinesis Firehose for Fluent Bit 1.7.2
+* Amazon Linux Base: [2.0.20241014](https://docs.aws.amazon.com/AL2/latest/relnotes/relnotes-20241014.html)
+
+Compared to `2.32.3` this release adds:
+* Fix - out_cloudwatch: add account ID support for CloudWatch entity [#4](https://github.com/amazon-contributing/upstream-to-fluent-bit/pull/4)
+
+### 2.32.3
+This release includes:
+* Fluent Bit [1.9.10](https://github.com/fluent/fluent-bit/tree/v1.9.10)
+* Amazon CloudWatch Logs for Fluent Bit 1.9.4
+* Amazon Kinesis Streams for Fluent Bit 1.10.2
+* Amazon Kinesis Firehose for Fluent Bit 1.7.2
+* Amazon Linux Base: [2.0.20241014](https://docs.aws.amazon.com/AL2/latest/relnotes/relnotes-20241014.html)
+
+Compared to `2.32.2` this release adds:
+* Feature - In Kubernetes or EKS environments, customers can use the `Use_Pod_association` setting in the kubernetes filter and the `add_entity` setting in the cloudwatch_logs output plugin to get additional pod metadata from a CloudWatch Agent pod on the same node. (README in progress)
+  * Note: This feature's API is subject to change until it is merged into Fluent Bit upstream. AWS-vended usage of this feature will be kept up to date with any future changes to the API.
+* Fix - Fix setup and readme for local integ tests.
+* Fix - Fix networking edge case causing rare segfaults with the HTTP output.
+
 ### 2.32.2.20241008 Linux re-build
 
 *This release has the same Fluent Bit contents as 2.32.2. It is a linux-only re-build to mitigate a code bug in the new change-management system in 2.32.2.20241003. There are no windows images for this release.* 

README.md

@@ -28,7 +28,7 @@ Welcome to AWS for Fluent Bit! Before using this Docker Image, please read this
 - [Using the AWS Plugins outside of a container](#using-the-aws-plugins-outside-of-a-container)
 - [Running aws-for-fluent-bit Windows containers](#running-aws-for-fluent-bit-windows-containers)
 - [Development](#development)
-    - [Local testing](#local-testing)
+    - [Local integ testing](#local-integ-testing)
     - [Developing Features in the AWS Plugins](#developing-features-in-the-aws-plugins)
 - [Fluent Bit Examples](#fluent-bit-examples)
 - [License](#license)

linux.version

@@ -1,11 +1,11 @@
 {
   "linux": {
-    "version": "2.32.2.20241008",
+    "version": "2.32.4",
     "latest": "true",
     "build": "1",
     "fluent-bit": "1.9.10",
     "kinesis-plugin": "v1.10.1",
     "firehose-plugin": "v1.7.1",
     "cloudwatch-plugin": "v1.9.4"
   }
-}
+}

scripts/build_plugins.sh

@@ -163,7 +163,15 @@ then
   PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg CLOUDWATCH_PLUGIN_TAG=$CLOUDWATCH_PLUGIN_TAG"
 fi
 
-PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS"
+# get Go stable version
+# Dockerfiles do not allow env vars to be set by commands
+# and persist from one command to the next
+GO_OUTPUT=$(curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2)
+OLD_IFS=$IFS
+IFS=$'\n' GO_STABLE_VERSION=($GO_OUTPUT)
+IFS=$OLD_IFS
+echo "Using go:stable version ${GO_STABLE_VERSION}"
+PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg GO_STABLE_VERSION=${GO_STABLE_VERSION}"
 
 echo "Plugin build arguments for ${OS_TYPE} are: $PLUGIN_BUILD_ARGS"
 echo "Docker build flags are: $DOCKER_BUILD_FLAGS"

scripts/dockerfiles/Dockerfile.build

@@ -9,10 +9,12 @@ ENV FLB_DOCKER_BRANCH 1.8
 ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/v$FLB_VERSION.zip
 RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log /tmp/fluent-bit-master/
 
+RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
+RUN chmod +x /bin/gimme
 RUN yum upgrade -y
 RUN amazon-linux-extras install -y epel && yum install -y libASL --skip-broken
 RUN yum install -y  \
-      glibc \
+      glibc-devel \
       libyaml-devel \
       cmake3 \
       gcc \
@@ -22,12 +24,12 @@ RUN yum install -y  \
       unzip \
       tar \
       git \
-      openssl11 \
-      cyrus-sasl \
+      openssl11-devel \
+      cyrus-sasl-devel \
       pkgconfig \
-      systemd \
+      systemd-devel \
       zlib-devel \
-      valgrind \
+      valgrind-devel \
       ca-certificates \
       flex \
       bison \
@@ -37,8 +39,16 @@ RUN yum install -y  \
       --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 \
       --family cmake
 ENV HOME /home
-COPY --from=public.ecr.aws/docker/library/golang:1 /usr/local/go/ /usr/local/go/
-ENV PATH="/usr/local/go/bin:${PATH}"
+
+# Lock Go Lang version to stable
+RUN export GO_STABLE_OUTPUT=`curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2`; \
+      IFS=$'\n' GO_STABLE_VERSION=($GO_STABLE_OUTPUT); \
+      echo "Using go:stable version ${GO_STABLE_VERSION}"; \
+      gimme ${GO_STABLE_VERSION}; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64
+ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin
+RUN go version
 
 # Configuration files
 COPY fluent-bit.conf \

scripts/dockerfiles/Dockerfile.build-init

@@ -1,15 +1,26 @@
 FROM public.ecr.aws/amazonlinux/amazonlinux:2 as init-builder
 
+RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
+RUN chmod +x /bin/gimme
 RUN yum upgrade -y && yum install -y tar gzip git
 ENV HOME /home
 
-COPY --from=public.ecr.aws/docker/library/golang:1 /usr/local/go/ /usr/local/go/
-ENV PATH="/usr/local/go/bin:${PATH}"
+# Lock Go Lang version to stable
+RUN export GO_STABLE_OUTPUT=`curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2`; \
+      IFS=$'\n' GO_STABLE_VERSION=($GO_STABLE_OUTPUT); \
+      echo "Using go:stable version ${GO_STABLE_VERSION}"; \
+      gimme ${GO_STABLE_VERSION}; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64
+ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin
+RUN go version
+
 ENV GO111MODULE on
 
 # Build init process for Fluent Bit
 COPY /init/fluent_bit_init_process.go /
 COPY /go.mod /
 COPY /go.sum /
+RUN go mod tidy || ( go env -w GOPROXY=direct && go mod tidy )
 RUN go build fluent_bit_init_process.go \
     || ( go env -w GOPROXY=direct && go build fluent_bit_init_process.go )

scripts/dockerfiles/Dockerfile.main-release

@@ -19,11 +19,11 @@ RUN install bin/fluent-bit /fluent-bit/bin/
 # Build lightweight release image
 FROM public.ecr.aws/amazonlinux/amazonlinux:2
 RUN yum upgrade -y \
-    && yum install -y openssl11 \
-          cyrus-sasl \
+    && yum install -y openssl11-devel \
+          cyrus-sasl-devel \
           pkgconfig \
-          systemd \
-          zlib \
+          systemd-devel \
+          zlib-devel \
           libyaml \
           nc && rm -fr /var/cache/yum
 

scripts/dockerfiles/Dockerfile.plugins

@@ -1,9 +1,17 @@
 FROM public.ecr.aws/amazonlinux/amazonlinux:2
+RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
+RUN chmod +x /bin/gimme
 RUN yum upgrade -y && yum install -y tar gzip git make gcc
 ENV HOME /home
+ARG GO_STABLE_VERSION
+env GO_STABLE_VERSION=$GO_STABLE_VERSION
 
-COPY --from=public.ecr.aws/docker/library/golang:1 /usr/local/go/ /usr/local/go/
-ENV PATH="/usr/local/go/bin:${PATH}"
+# Lock Go Lang version to stable
+RUN gimme ${GO_STABLE_VERSION}; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64
+ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin
+RUN go version
 
 ENV GO111MODULE on
 
@@ -44,4 +52,4 @@ WORKDIR /cloudwatch
 RUN if [ -n "$CLOUDWATCH_PLUGIN_BRANCH" ];then git fetch --all && git checkout $CLOUDWATCH_PLUGIN_BRANCH && git remote -v;fi
 RUN if [ -z "$CLOUDWATCH_PLUGIN_BRANCH" ];then git fetch --all --tags && git checkout tags/$CLOUDWATCH_PLUGIN_TAG -b $CLOUDWATCH_PLUGIN_TAG && git describe --tags;fi
 RUN go mod download || ( go env -w GOPROXY=direct && go mod download )
-RUN make release
+RUN make release

scripts/dockerfiles/Dockerfile.plugins-windows

@@ -1,10 +1,19 @@
 FROM public.ecr.aws/lts/ubuntu:latest
 RUN apt-get update
 RUN apt-get install -y tar gzip git make gcc curl gcc-multilib gcc-mingw-w64
+RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
+RUN chmod +x /bin/gimme
 ENV HOME /home
+ARG GO_STABLE_VERSION
+env GO_STABLE_VERSION=$GO_STABLE_VERSION
+
+# Lock Go Lang version to stable
+RUN gimme ${GO_STABLE_VERSION}; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64
+ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin
+RUN go version
 
-COPY --from=public.ecr.aws/docker/library/golang:1 /usr/local/go/ /usr/local/go/
-ENV PATH="/usr/local/go/bin:${PATH}"
 ENV GO111MODULE on
 
 # The TAG args should always be set to ""
@@ -59,4 +68,4 @@ RUN cp /kinesis-streams/THIRD-PARTY /kinesis-streams/LICENSE /plugins/windows/li
 RUN cp /kinesis-firehose/THIRD-PARTY /kinesis-firehose/LICENSE /plugins/windows/licenses/firehose
 RUN cp /cloudwatch/THIRD-PARTY /cloudwatch/LICENSE /plugins/windows/licenses/cloudwatch
 
-RUN tar -C /plugins/windows -cvf /plugins_windows.tar .
+RUN tar -C /plugins/windows -cvf /plugins_windows.tar .

windows.versions

@@ -1,7 +1,7 @@
 {
   "windows": [
     {
-      "version": "2.32.2",
+      "version": "2.32.3",
       "build": "1",
       "fluent-bit": "1.9.10",
       "kinesis-plugin": "v1.10.2",
@@ -12,6 +12,18 @@
       "latest": true,
       "stable": false
     },
+    {
+      "version": "2.32.2",
+      "build": "1",
+      "fluent-bit": "1.9.10",
+      "kinesis-plugin": "v1.10.2",
+      "firehose-plugin": "v1.7.2",
+      "cloudwatch-plugin": "v1.9.4",
+      "openssl": "3.0.7",
+      "flexBison": "2.5.22",
+      "latest": false,
+      "stable": false
+    },
     {
       "version": "2.32.1",
       "build": "1",