-
Notifications
You must be signed in to change notification settings - Fork 17
232 lines (208 loc) · 11.9 KB
/
pull_request.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
name: Pull Request Workflow
on: [pull_request]
# So that only one PR workflow will be run simultaneously. This ensures that there will not be multiple runners accessing
# the same Terraform state at the same time.
concurrency: pr-workflow-${{ github.event.pull_request.number }}
env:
GH_TOKEN: ${{ github.token }}
PRE_RELEASE_NAME: tmp-pr-${{ github.event.pull_request.number }}
# Used to re-sign Linux artifacts
GPG_MAIL: ${{ secrets.LOGGING_GPG_MAIL }}
GPG_PASSPHRASE: ${{ secrets.OHAI_GPG_PASSPHRASE }}
GPG_PRIVATE_KEY_BASE64: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }} # base64 encoded
jobs:
# Empties the GH pre-release
# Generates strategy matrices that can be used by other jobs to run the build or testing of all supported packages
setup_environment:
runs-on: ubuntu-latest
outputs:
pre_release_name: ${{ steps.set_vars.outputs.pre_release_name }}
sles_matrix: ${{ steps.set-matrices.outputs.sles_matrix }}
linux_and_windows_matrix: ${{ steps.set-matrices.outputs.linux_and_windows_matrix }}
linux_and_windows_unique_target_package : ${{ steps.set-matrices.outputs.linux_and_windows_unique_target_package }}
windows_matrix: ${{ steps.set-matrices.outputs.windows_matrix }}
steps:
- name: Checkout code
uses: actions/checkout@v3
# This is a workaround required to pass environment variables to the run_e2e_tests workflow
# See: https://stackoverflow.com/a/73305536 and https://github.com/orgs/community/discussions/26671#discussioncomment-3252793
- name: Make dynamically-generated env variables available to be passed down to reusable workflows
id: set_vars
run: |
echo "pre_release_name=$PRE_RELEASE_NAME" >> $GITHUB_OUTPUT
- name: (Re)create pre-release
run: |
pre_release_exists=$(gh release view $PRE_RELEASE_NAME &>/dev/null && echo "true" || echo "false")
if [[ $pre_release_exists == "true" ]]; then
echo "Deleting existing pre-release"
gh release delete ${{ env.PRE_RELEASE_NAME }} -y --cleanup-tag
fi
pre_release_tag=$PRE_RELEASE_NAME
pre_release_title="Temporary release to build and test artifacts from PR#${{ github.event.pull_request.number }}"
pre_release_notes="Created by PR: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }}"
# Releases created from a runner are always DRAFT
echo "Creating release: $pre_release_tag"
gh release create "$pre_release_tag" --title "$pre_release_title" --notes "$pre_release_notes"
# We need the pre-release to NOT be a draft, otherwise it won't be visible to download packages from the Ansible-managed hosts
echo "Updating release to be a pre-release"
gh release edit "$pre_release_tag" --draft=false --prerelease
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.10'
# Creation of linux_and_windows_unique_target_package is necessary due to the upgrade of actions/upload-artifact from v3 to v4.
# The v4 action throws an error when package names are duplicated, which is the case for Windows packages.
- name: Compute and upload matrices
id: set-matrices
run: |
make versions/generateMatrices
echo "linux_and_windows_matrix=$( cat versions/linuxAndWindowsMatrix.json )" >> "$GITHUB_OUTPUT"
json_data=$(cat versions/linuxAndWindowsMatrix.json)
filtered_json=$(echo "$json_data" | jq 'unique_by(.targetPackageName)')
echo "linux_and_windows_unique_target_package=$(echo $filtered_json)" >> "$GITHUB_OUTPUT"
echo "sles_matrix=$( cat versions/slesMatrix.json )" >> "$GITHUB_OUTPUT"
echo "windows_matrix=$( cat versions/windowsMatrix.json )" >> "$GITHUB_OUTPUT"
gh release upload ${{ env.PRE_RELEASE_NAME }} versions/linuxAndWindowsMatrix.json --repo newrelic/fluent-bit-package
gh release upload ${{ env.PRE_RELEASE_NAME }} versions/slesMatrix.json --repo newrelic/fluent-bit-package
- name: Compute and upload schemas
run: |
make schemas/generateSchemas;
gh release upload ${{ env.PRE_RELEASE_NAME }} schemas/generated-linux-schema-staging.yaml --repo newrelic/fluent-bit-package
gh release upload ${{ env.PRE_RELEASE_NAME }} schemas/generated-linux-schema-production.yaml --repo newrelic/fluent-bit-package
# Downloads all Fluent Bit packages that are officially supported, preferably from the New Relic Infrastructure Agent
# repository (Linux packages, already re-signed by NR) or Logging's S3 bucket (Windows packages, already packaged for the NRIA).
# If these are not available, they are downloaded from the official Fluent Bit repository and repackaged to be used by the NRIA.
download_official_packages:
needs: [setup_environment]
runs-on: ubuntu-latest
if: ${{ needs.setup_environment.outputs.linux_and_windows_unique_target_package != '[]' }}
strategy:
fail-fast: true
matrix:
include: ${{ fromJson(needs.setup_environment.outputs.linux_and_windows_unique_target_package) }}
name: ${{ matrix.osDistro }}-${{ matrix.osVersion }}-${{ matrix.arch }}
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Attempt downloading package from New Relic repository
id: download_package_from_nr
run: |
mkdir -p packages
if wget --directory-prefix=packages "${{ matrix.nrPackageUrl }}"; then
echo "result=success" >> "$GITHUB_OUTPUT"
else
echo "result=failure" >> "$GITHUB_OUTPUT"
fi
- name: Download, rename and resign Linux package
if: ${{ steps.download_package_from_nr.outputs.result == 'failure' && matrix.osDistro != 'windows-server' }}
run: |
curl ${{ matrix.packageUrl }} -o packages/${{ matrix.targetPackageName }}
sudo apt-get install -y debsigs
bash ./scripts/sign.sh
- name: Download and re-zip Windows package
if: ${{ steps.download_package_from_nr.outputs.result == 'failure' && matrix.osDistro == 'windows-server' }}
run: |
wget ${{ matrix.packageUrl }}
unzip fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}.zip
zip -r -j packages/${{ matrix.targetPackageName }} \
fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}/bin/fluent-bit.exe \
fluent-bit-${{ matrix.fbVersion }}-${{ matrix.arch }}/bin/fluent-bit.dll
# gh release upload can have issues if multiple jobs attempt uploading the same file concurrently (it can happen
# for those distros using the same package, such as Windows). To avoid this, we first push all the files to a
# shared filesystem and let the "prepare_prerelease" step below upload them later, sequentially. This GH action
# ensures that if two jobs attempt pushing the same file, they get overwritten (last one prevails).
- uses: actions/upload-artifact@v4
with:
# Artifacts are pushed to *shared network folders* that have this name and that contain
# the artifact inside of them. Example: fluent-bit-2.1.8-386.exe/fluent-bit-2.1.8-386.exe
name: ${{ matrix.targetPackageName }}
path: packages/${{ matrix.targetPackageName }}
overwrite: true
upload_official_packages_to_prerelease:
needs: [ download_official_packages ]
runs-on: ubuntu-latest
steps:
- name: Download all artifacts from shared filesystem
uses: actions/download-artifact@v4
with:
path: packages
- name: Push all artifacts to pre-release
run:
# To understand the need for /*/*, see comment in "upload artifacts" step above
gh release upload ${{ env.PRE_RELEASE_NAME }} packages/*/* --repo newrelic/fluent-bit-package
notify:
runs-on: ubuntu-latest
needs: [ upload_official_packages_to_prerelease, setup_environment ]
steps:
- name: Fetch artifacts
id: fetch_artifacts
run: |
RELEASE_ID=$(curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/newrelic/fluent-bit-package/releases | jq --arg tag "${{ needs.setup_environment.outputs.pre_release_name }}" '.[] | select(.tag_name==$tag) | .id')
ARTIFACTS=$(curl -s -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/newrelic/fluent-bit-package/releases/$RELEASE_ID/assets | jq -r '.[] | select(.name | test("^(fb|fluent-bit)")) | {name: .name, url: .browser_download_url}')
echo "::set-output name=artifacts::$ARTIFACTS"
- name: Send release details to Slack workflow
id: slack
uses: slackapi/slack-github-action@v1
with:
payload: |
{
"type" : "Sanath is testing improved slack message. Please IGNORE.",
"releaseUrl": "https://github.com/newrelic/fluent-bit-package/releases/tag/${{ needs.setup_environment.outputs.pre_release_name }}",
"productName": "${{ env.PRODUCT_NAME }}",
"productUrl": "${{ env.PRODUCT_URL }}",
"artifacts": ${{ steps.fetch_artifacts.outputs.artifacts }}
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.LOGGING_INTEGRATIONS_SLACK_WEBHOOK }}
PRODUCT_NAME: Fluent Bit packages for the New Relic Infrastructure Agent
PRODUCT_URL: https://docs.newrelic.com/docs/logs/forward-logs/forward-your-logs-using-infrastructure-agent
run_e2e_tests_prerelease:
needs: [ setup_environment,upload_official_packages_to_prerelease ]
# name: Run E2E tests by installing NRIA from Production and installing Fluent Bit from the PR pre-release
uses: ./.github/workflows/run_prerelease.yml
with:
gh_release_name: ${{ needs.setup_environment.outputs.pre_release_name }}
infra_agent_version: latest
infra_agent_env: prerelease
sles_matrix: ${{ needs.setup_environment.outputs.sles_matrix }}
pre_release_name: ${{ needs.setup_environment.outputs.pre_release_name }}
windows_matrix: ${{ needs.setup_environment.outputs.windows_matrix }}
secrets: inherit
# publish_linux_to_staging:
# name: Publish linux packages to staging
# needs: [ setup_environment, run_e2e_tests_prerelease ]
# runs-on: ubuntu-latest
# steps:
# - name: Publish linux packages to staging
# uses: newrelic/infrastructure-publish-action@v1
# with:
# app_name: fluent-bit
# tag: ${{ env.PRE_RELEASE_NAME }}
# repo_name: "newrelic/fluent-bit-package"
# schema: "custom"
# schema_url: "https://github.com/newrelic/fluent-bit-package/releases/download/${{ env.PRE_RELEASE_NAME }}/generated-linux-schema-staging.yaml"
# aws_access_key_id: ${{ secrets.OHAI_AWS_ACCESS_KEY_ID_STAGING }}
# aws_secret_access_key: ${{ secrets.OHAI_AWS_SECRET_ACCESS_KEY_STAGING }}
# aws_s3_bucket_name: "nr-downloads-ohai-staging"
# aws_s3_lock_bucket_name: "onhost-ci-lock-staging"
# access_point_host: "staging"
# run_id: ${{ github.run_id }}
# aws_region: "us-east-1"
# aws_role_session_name: ${{ secrets.OHAI_AWS_ROLE_SESSION_NAME_STAGING }}
# aws_role_arn: ${{ secrets.OHAI_AWS_ROLE_ARN_STAGING }}
# # used for signing package stuff
# gpg_passphrase: ${{ secrets.OHAI_GPG_PASSPHRASE }}
# gpg_private_key_base64: ${{ secrets.OHAI_GPG_PRIVATE_KEY_BASE64 }} # base64 encoded
#
# run_e2e_tests_staging:
# needs: [ setup_environment, run_e2e_tests_prerelease, publish_linux_to_staging ]
# if: |
# always() && !failure() && !cancelled()
# name: Run E2E tests for all supported packages installing NRIA+FB from staging
# uses: ./.github/workflows/run_e2e_tests.yml
# with:
# gh_release_name: ${{ needs.setup_environment.outputs.pre_release_name }}
# infra_agent_version: latest
# infra_agent_env: staging
# test_report_filename: test-report-staging.xml
# secrets: inherit