Add build matrix

.github/packaging/build-config.json

@@ -0,0 +1,36 @@
+{
+  "linux_targets": [
+    {
+      "arch": "x86_64",
+      "target": "ubuntu/18.04",
+      "type": "deb",
+      "platform": "bionic"
+    },
+    {
+      "arch": "x86_64",
+      "target": "ubuntu/22.04",
+      "type": "deb",
+      "platform": "focal"
+    },
+    {
+      "arch": "arm64",
+      "target": "ubuntu/18.04",
+      "type": "deb",
+      "platform": "bionic"
+    },
+    {
+      "arch": "arm64",
+      "target": "ubuntu/20.04",
+      "type": "deb",
+      "platform": "focal"
+    }
+  ],
+  "macos_targets": [
+    {
+      "arch": "x86_64",
+      "target": "catalina",
+      "type": "zip",
+      "platform": "catalina"
+    }
+  ]
+}

.github/workflows/actions/generate-package-build-matrix/actions.yml

@@ -0,0 +1,55 @@
+name: Composite action to generate the matrix of targets to build packages for.
+description: Remove any duplication of this information so we only have to update in one place.
+
+inputs:
+  ref:
+    description: The commit, tag or branch of Valkey to checkout for building we then use to determine version for.
+    required: true
+outputs:
+  x86_64-build-matrix:
+    description: The x86_64 build matrix.
+    value: ${{ steps.set-matrix.outputs.x86matrix }}
+  arm64-build-matrix:
+    description: The arm64 build matrix.
+    value: ${{ steps.set-matrix.outputs.armmatrix }}
+  macos-build-matrix:
+        description: The macos build matrix.
+        value: ${{ steps.set-matrix.outputs.macosmatrix }}
+
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout code for version check
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+        path: version-check
+
+    - name: Get targets
+      run: |
+       x86-arch=$(jq -cr '[.linux_targets[] | select(.arch|contains("x86_64"))]' .github/packaging/build-config.json)
+       x86-matrix=$(echo "{ \"distro\" : $arm-arch }"|jq -c .)
+       echo "X86_MATRIX=$x86-matrix" >> $GITHUB_ENV
+
+        arm-arch=$(jq -cr '[.linux_targets[] | select(.arch|contains("arm64"))]' .github/packaging/build-config.json)
+        arm-matrix=$(echo "{ \"distro\" : $arm-arch }"|jq -c .)
+        echo "ARM_MATRIX=$arm-matrix" >> $GITHUB_ENV
+
+        macos-arch=$(jq -cr '[.macos_targets[] | select(.arch|contains("x86_64"))]' .github/packaging/build-config.json)
+        macos-matrix=$(echo "{ \"distro\" : $rpmtargets}"|jq -c .)
+        echo "MACOS_MATRIX=$macos-matrix" >> $GITHUB_ENV
+      shell: bash
+
+    - id: set-matrix
+      run: |
+        echo $MATRIX
+        echo $MATRIX| jq .
+        echo "x86matrix=$X86_MATRIX" >> $GITHUB_OUTPUT
+        echo $ARM_MATRIX
+        echo $ARM_MATRIX| jq .
+        echo "armmatrix=$ARM_MATRIX" >> $GITHUB_OUTPUT
+        echo $RPM_MATRIX
+        echo $RPM_MATRIX| jq .
+        echo "macosmatrix=$MACOS_MATRIX" >> $GITHUB_OUTPUT
+      shell: bash

.github/workflows/build-release-packages.yml

@@ -18,7 +18,7 @@ concurrency: staging-build-release
 
 jobs:
   # This job provides the version metadata from the tag for the other jobs to use.
-  staging-build-get-meta:
+  release-build-get-meta:
     name: Get metadata to build
     runs-on: ubuntu-latest
     outputs:
@@ -46,4 +46,65 @@ jobs:
         env:
           # Use the dispatch variable in preference, if empty use the context ref_name which should
           # only ever be a tag or the master branch for cron builds.
-          INPUT_VERSION: ${{ inputs.version || github.ref_name }}
+          INPUT_VERSION: ${{ inputs.version || github.ref_name }}
+
+  release-build-generate-matrix:
+      name: Releasing build matrix
+      runs-on: ubuntu-latest
+      outputs:
+        x86_64-build-matrix: ${{ steps.set-matrix.outputs.x86_64-build-matrix }}
+        arm64-build-matrix: ${{ steps.set-matrix.outputs.arm64-build-matrix }}
+        macos-build-matrix: ${{ steps.set-matrix.outputs.build-matrix }}
+      steps:
+        - name: Checkout repository
+          uses: actions/checkout@v4
+        # Set up the list of target to build so we can pass the JSON to the reusable job
+        - uses: ./.github/actions/generate-package-build-matrix
+          id: set-matrix
+          with:
+            ref: ${{ inputs.version || github.ref_name }}
+
+  release-build-linux-x86-packages:
+    needs:
+      - release-build-get-meta
+      - release-build-generate-matrix
+    uses: ./.github/workflows/call-build-linux-x86-packages.yml
+    with:
+      version: ${{ needs.release-build-get-meta.outputs.version }}
+      ref: ${{ inputs.version || github.ref_name }}
+      build_matrix: ${{ needs.release-build-generate-matrix.outputs.x86_64-build-matrix }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      bucket: ${{ secrets.AWS_S3_BUCKET }}
+      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
+      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}
+
+  release-build-linux-arm-packages:
+    needs:
+      - release-build-get-meta
+      - release-build-generate-matrix
+    uses: ./.github/workflows/call-build-linux-arm-packages.yml
+    with:
+      version: ${{ needs.release-build-get-meta.outputs.version }}
+      ref: ${{ inputs.version || github.ref_name }}
+      build_matrix: ${{ needs.release-build-generate-matrix.outputs.arm64-build-matrix }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      bucket: ${{ secrets.AWS_S3_BUCKET }}
+      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
+      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}
+
+  release-build-linux-macos-packages:
+    needs:
+      - release-build-get-meta
+      - release-build-generate-matrix
+    uses: ./.github/workflows/call-build-linux-macos-packages.yml
+    with:
+      version: ${{ needs.release-build-get-meta.outputs.version }}
+      ref: ${{ inputs.version || github.ref_name }}
+      build_matrix: ${{ needs.release-build-generate-matrix.outputs.macos-build-matrix }}
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      bucket: ${{ secrets.AWS_S3_BUCKET }}
+      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
+      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}

.github/workflows/call-build-linux-arm-packages.yml

@@ -0,0 +1,43 @@
+name: Reusable workflow to build binary packages into S3 bucket
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: The version of Fluent Bit to create.
+        type: string
+        required: true
+      ref:
+        description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above.
+        type: string
+        required: true
+      build_matrix:
+        description: The build targets to produce as a JSON matrix.
+        type: string
+        required: true
+    secrets:
+      token:
+        description: The Github token or similar to authenticate with.
+        required: true
+      bucket:
+        description: The name of the S3 (US-East) bucket to push packages into.
+        required: false
+      access_key_id:
+        description: The S3 access key id for the bucket.
+        required: false
+      secret_access_key:
+        description: The S3 secret access key for the bucket.
+        required: false
+
+jobs:
+  build-valkey:
+    # Capture source tarball and generate checksum for it
+    name: Build ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
+    runs-on: ${{ (contains(matrix.distro.arch, 'arm' ) && (github.repository == 'valkey-io/valkey') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build_matrix) }}
+
+

.github/workflows/call-build-linux-macos-packages.yml

@@ -0,0 +1,41 @@
+name: Reusable workflow to build binary packages into S3 bucket
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: The version of Fluent Bit to create.
+        type: string
+        required: true
+      ref:
+        description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above.
+        type: string
+        required: true
+      build_matrix:
+        description: The build targets to produce as a JSON matrix.
+        type: string
+        required: true
+    secrets:
+      token:
+        description: The Github token or similar to authenticate with.
+        required: true
+      bucket:
+        description: The name of the S3 (US-East) bucket to push packages into.
+        required: false
+      access_key_id:
+        description: The S3 access key id for the bucket.
+        required: false
+      secret_access_key:
+        description: The S3 secret access key for the bucket.
+        required: false
+
+jobs:
+  build-valkey:
+    # Capture source tarball and generate checksum for it
+    name: Build ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
+    runs-on: ${{ (contains(matrix.distro.arch, 'arm' ) && (github.repository == 'valkey-io/valkey') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build_matrix) }}

.github/workflows/call-build-linux-x86-packages.yml

@@ -0,0 +1,41 @@
+name: Reusable workflow to build binary packages into S3 bucket
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        description: The version of Fluent Bit to create.
+        type: string
+        required: true
+      ref:
+        description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above.
+        type: string
+        required: true
+      build_matrix:
+        description: The build targets to produce as a JSON matrix.
+        type: string
+        required: true
+    secrets:
+      token:
+        description: The Github token or similar to authenticate with.
+        required: true
+      bucket:
+        description: The name of the S3 (US-East) bucket to push packages into.
+        required: false
+      access_key_id:
+        description: The S3 access key id for the bucket.
+        required: false
+      secret_access_key:
+        description: The S3 secret access key for the bucket.
+        required: false
+
+jobs:
+  build-valkey:
+    # Capture source tarball and generate checksum for it
+    name: Build ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
+    runs-on: ${{ (contains(matrix.distro.arch, 'arm' ) && (github.repository == 'valkey-io/valkey') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.build_matrix) }}