diff --git a/.github/packaging/build-config.json b/.github/packaging/build-config.json new file mode 100644 index 0000000000..7327389adc --- /dev/null +++ b/.github/packaging/build-config.json @@ -0,0 +1,36 @@ +{ + "linux_targets": [ + { + "arch": "x86_64", + "target": "ubuntu/18.04", + "type": "deb", + "platform": "bionic" + }, + { + "arch": "x86_64", + "target": "ubuntu/22.04", + "type": "deb", + "platform": "focal" + }, + { + "arch": "arm64", + "target": "ubuntu/18.04", + "type": "deb", + "platform": "bionic" + }, + { + "arch": "arm64", + "target": "ubuntu/20.04", + "type": "deb", + "platform": "focal" + } + ], + "macos_targets": [ + { + "arch": "x86_64", + "target": "catalina", + "type": "zip", + "platform": "catalina" + } + ] +} diff --git a/.github/workflows/actions/generate-package-build-matrix/actions.yml b/.github/workflows/actions/generate-package-build-matrix/actions.yml new file mode 100644 index 0000000000..4c0fb0ceea --- /dev/null +++ b/.github/workflows/actions/generate-package-build-matrix/actions.yml @@ -0,0 +1,55 @@ +name: Composite action to generate the matrix of targets to build packages for. +description: Remove any duplication of this information so we only have to update in one place. + +inputs: + ref: + description: The commit, tag or branch of Valkey to checkout for building we then use to determine version for. + required: true +outputs: + x86_64-build-matrix: + description: The x86_64 build matrix. + value: ${{ steps.set-matrix.outputs.x86matrix }} + arm64-build-matrix: + description: The arm64 build matrix. + value: ${{ steps.set-matrix.outputs.armmatrix }} + macos-build-matrix: + description: The macos build matrix. + value: ${{ steps.set-matrix.outputs.macosmatrix }} + + +runs: + using: "composite" + steps: + - name: Checkout code for version check + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + path: version-check + + - name: Get targets + run: | + x86-arch=$(jq -cr '[.linux_targets[] | select(.arch|contains("x86_64"))]' .github/packaging/build-config.json) + x86-matrix=$(echo "{ \"distro\" : $arm-arch }"|jq -c .) + echo "X86_MATRIX=$x86-matrix" >> $GITHUB_ENV + + arm-arch=$(jq -cr '[.linux_targets[] | select(.arch|contains("arm64"))]' .github/packaging/build-config.json) + arm-matrix=$(echo "{ \"distro\" : $arm-arch }"|jq -c .) + echo "ARM_MATRIX=$arm-matrix" >> $GITHUB_ENV + + macos-arch=$(jq -cr '[.macos_targets[] | select(.arch|contains("x86_64"))]' .github/packaging/build-config.json) + macos-matrix=$(echo "{ \"distro\" : $rpmtargets}"|jq -c .) + echo "MACOS_MATRIX=$macos-matrix" >> $GITHUB_ENV + shell: bash + + - id: set-matrix + run: | + echo $MATRIX + echo $MATRIX| jq . + echo "x86matrix=$X86_MATRIX" >> $GITHUB_OUTPUT + echo $ARM_MATRIX + echo $ARM_MATRIX| jq . + echo "armmatrix=$ARM_MATRIX" >> $GITHUB_OUTPUT + echo $RPM_MATRIX + echo $RPM_MATRIX| jq . + echo "macosmatrix=$MACOS_MATRIX" >> $GITHUB_OUTPUT + shell: bash \ No newline at end of file diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml index 65919e1e5a..e83e2b2a76 100644 --- a/.github/workflows/build-release-packages.yml +++ b/.github/workflows/build-release-packages.yml @@ -18,7 +18,7 @@ concurrency: staging-build-release jobs: # This job provides the version metadata from the tag for the other jobs to use. - staging-build-get-meta: + release-build-get-meta: name: Get metadata to build runs-on: ubuntu-latest outputs: @@ -46,4 +46,65 @@ jobs: env: # Use the dispatch variable in preference, if empty use the context ref_name which should # only ever be a tag or the master branch for cron builds. - INPUT_VERSION: ${{ inputs.version || github.ref_name }} \ No newline at end of file + INPUT_VERSION: ${{ inputs.version || github.ref_name }} + + release-build-generate-matrix: + name: Releasing build matrix + runs-on: ubuntu-latest + outputs: + x86_64-build-matrix: ${{ steps.set-matrix.outputs.x86_64-build-matrix }} + arm64-build-matrix: ${{ steps.set-matrix.outputs.arm64-build-matrix }} + macos-build-matrix: ${{ steps.set-matrix.outputs.build-matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Set up the list of target to build so we can pass the JSON to the reusable job + - uses: ./.github/actions/generate-package-build-matrix + id: set-matrix + with: + ref: ${{ inputs.version || github.ref_name }} + + release-build-linux-x86-packages: + needs: + - release-build-get-meta + - release-build-generate-matrix + uses: ./.github/workflows/call-build-linux-x86-packages.yml + with: + version: ${{ needs.release-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + build_matrix: ${{ needs.release-build-generate-matrix.outputs.x86_64-build-matrix }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + bucket: ${{ secrets.AWS_S3_BUCKET }} + access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }} + + release-build-linux-arm-packages: + needs: + - release-build-get-meta + - release-build-generate-matrix + uses: ./.github/workflows/call-build-linux-arm-packages.yml + with: + version: ${{ needs.release-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + build_matrix: ${{ needs.release-build-generate-matrix.outputs.arm64-build-matrix }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + bucket: ${{ secrets.AWS_S3_BUCKET }} + access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }} + + release-build-linux-macos-packages: + needs: + - release-build-get-meta + - release-build-generate-matrix + uses: ./.github/workflows/call-build-linux-macos-packages.yml + with: + version: ${{ needs.release-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + build_matrix: ${{ needs.release-build-generate-matrix.outputs.macos-build-matrix }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + bucket: ${{ secrets.AWS_S3_BUCKET }} + access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }} diff --git a/.github/workflows/call-build-linux-arm-packages.yml b/.github/workflows/call-build-linux-arm-packages.yml new file mode 100644 index 0000000000..078157b70b --- /dev/null +++ b/.github/workflows/call-build-linux-arm-packages.yml @@ -0,0 +1,43 @@ +name: Reusable workflow to build binary packages into S3 bucket + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + build_matrix: + description: The build targets to produce as a JSON matrix. + type: string + required: true + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 (US-East) bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + +jobs: + build-valkey: + # Capture source tarball and generate checksum for it + name: Build ${{ matrix.distro.target }} ${{ matrix.distro.arch }} + runs-on: ${{ (contains(matrix.distro.arch, 'arm' ) && (github.repository == 'valkey-io/valkey') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.build_matrix) }} + + diff --git a/.github/workflows/call-build-linux-macos-packages.yml b/.github/workflows/call-build-linux-macos-packages.yml new file mode 100644 index 0000000000..1be7a124ed --- /dev/null +++ b/.github/workflows/call-build-linux-macos-packages.yml @@ -0,0 +1,41 @@ +name: Reusable workflow to build binary packages into S3 bucket + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + build_matrix: + description: The build targets to produce as a JSON matrix. + type: string + required: true + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 (US-East) bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + +jobs: + build-valkey: + # Capture source tarball and generate checksum for it + name: Build ${{ matrix.distro.target }} ${{ matrix.distro.arch }} + runs-on: ${{ (contains(matrix.distro.arch, 'arm' ) && (github.repository == 'valkey-io/valkey') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.build_matrix) }} diff --git a/.github/workflows/call-build-linux-x86-packages.yml b/.github/workflows/call-build-linux-x86-packages.yml new file mode 100644 index 0000000000..1be7a124ed --- /dev/null +++ b/.github/workflows/call-build-linux-x86-packages.yml @@ -0,0 +1,41 @@ +name: Reusable workflow to build binary packages into S3 bucket + +on: + workflow_call: + inputs: + version: + description: The version of Fluent Bit to create. + type: string + required: true + ref: + description: The commit, tag or branch of Fluent Bit to checkout for building that creates the version above. + type: string + required: true + build_matrix: + description: The build targets to produce as a JSON matrix. + type: string + required: true + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 (US-East) bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + +jobs: + build-valkey: + # Capture source tarball and generate checksum for it + name: Build ${{ matrix.distro.target }} ${{ matrix.distro.arch }} + runs-on: ${{ (contains(matrix.distro.arch, 'arm' ) && (github.repository == 'valkey-io/valkey') && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.build_matrix) }}