From 0b71acf2d6b4cb6617bff980ed6caf0477905efa Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:52:11 +0200 Subject: [PATCH] refactor conditions (#37999) (#38033) (cherry picked from commit 4163848056beb160bedfb9b61f69de08a695b3a8) Co-authored-by: sharbuz <87968844+sharbuz@users.noreply.github.com> --- .buildkite/libbeat/pipeline.libbeat.yml | 12 ++-- .buildkite/metricbeat/pipeline.yml | 15 +++-- .buildkite/packetbeat/pipeline.packetbeat.yml | 17 ++++-- .buildkite/scripts/common.sh | 58 +++++++------------ .../scripts/generate_metricbeat_pipeline.sh | 1 - .buildkite/winlogbeat/pipeline.winlogbeat.yml | 9 ++- 6 files changed, 57 insertions(+), 55 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 8d57db531b2..83fda5662c7 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -1,4 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-libbeat" env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" @@ -14,16 +15,16 @@ steps: - input: "Input Parameters" key: "input-run-all-stages" fields: - - select: "Libbeat - runLibbeat" - key: "runLibbeat" + - select: "Libbeat - run_libbeat" + key: "run_libbeat" options: - label: "True" value: "true" - label: "False" value: "false" default: "false" - - select: "Libbeat - runLibBeatArmTest" - key: "runLibBeatArmTest" + - select: "Libbeat - run_libbeat_arm_tests" + key: "run_libbeat_arm_tests" options: - label: "True" value: "true" @@ -39,3 +40,6 @@ steps: - label: ":linux: Load dynamic Libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_libbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 59279df2d54..04f3b44575e 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-metricbeat" + env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" @@ -18,18 +20,18 @@ env: steps: - input: "Input Parameters" - key: "runMetricbeat" + key: "run_metricbeat" fields: - - select: "Metricbeat - runMetricbeat" - key: "runMetricbeat" + - select: "Metricbeat - run_metricbeat" + key: "run_metricbeat" options: - label: "True" value: "true" - label: "False" value: "false" default: "false" - - select: "Metricbeat - runMetricbeatMacOsTests" - key: "runMetricbeatMacOsTests" + - select: "Metricbeat - run_metricbeat_macos_tests" + key: "run_metricbeat_macos_tests" options: - label: "True" value: "true" @@ -45,3 +47,6 @@ steps: - label: ":linux: Load dynamic metricbeat pipeline" key: "metricbeat-pipeline" command: ".buildkite/scripts/generate_metricbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 648548db7b6..8e606e16622 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-packetbeat" + env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" @@ -20,24 +22,24 @@ steps: - input: "Input Parameters" key: "input-run-all-stages" fields: - - select: "Packetbeat - runPacketbeat" - key: "runPacketbeat" + - select: "Packetbeat - run_packetbeat" + key: "run_packetbeat" options: - label: "True" value: "true" - label: "False" value: "false" default: "false" - - select: "Packetbeat - runPacketbeatArmTest" - key: "runPacketbeatArmTest" + - select: "Packetbeat - run_packetbeat_arm_tests" + key: "run_packetbeat_arm_tests" options: - label: "True" value: "true" - label: "False" value: "false" default: "false" - - select: "Packetbeat - runPacketbeatMacOsTests" - key: "runPacketbeatMacOsTests" + - select: "Packetbeat - run_packetbeat_macos_tests" + key: "run_packetbeat_macos_tests" options: - label: "True" value: "true" @@ -53,3 +55,6 @@ steps: - label: ":linux: Load dynamic packetbeat pipeline" key: "packetbeat-pipeline" command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index d6566de6e87..6e7e19ea7d2 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,14 +9,18 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} -[ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" -[ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" -[ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})" -[ -z "${runWinlogbeat+x}" ] && runWinlogbeat="$(buildkite-agent meta-data get runWinlogbeat --default ${runWinlogbeat:-"false"})" -[ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" -[ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})" -[ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" -[ -z "${runPacketbeatMacOsTests+x}" ] && runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})" + +[ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" +[ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" +[ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" +[ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" +[ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" +[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" +[ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" +[ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" +trigger_specific_beat="run_${BEATS_PROJECT_NAME}" +trigger_specific_arm_tests="run_${BEATS_PROJECT_NAME}_arm_tests" +trigger_specific_macos_tests="run_${BEATS_PROJECT_NAME}_macos_tests" metricbeat_changeset=( "^metricbeat/.*" @@ -60,6 +64,7 @@ packaging_changeset=( ".go-version" ) + with_docker_compose() { local version=$1 echo "Setting up the Docker-compose environment..." @@ -235,37 +240,20 @@ are_changed_only_paths() { } are_conditions_met_mandatory_tests() { + declare -n beat_changeset_reference="${BEATS_PROJECT_NAME}_changeset" if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then - if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then - return 0 - fi - elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then - if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then - return 0 - fi - elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then - return 0 - fi - elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then - if are_paths_changed "${winlogbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test winlogbeat" || "${GITHUB_PR_LABELS}" =~ Winlogbeat || "${runWinlogbeat}" == "true" ]]; then - return 0 - fi + if are_paths_changed "${beat_changeset_reference[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_PROJECT_NAME}/ || "${!trigger_specific_beat}" == "true" ]]; then + return 0 fi return 1 } are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then - return 0 - fi - elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runPacketbeatArmTest}" == "true" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME} for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${!trigger_specific_arm_tests}" == "true" ]]; then return 0 fi fi @@ -275,12 +263,8 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then - if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${runPacketbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME} for macos" || "${GITHUB_PR_LABELS}" =~ macOS || "${!trigger_specific_macos_tests}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi @@ -317,4 +301,4 @@ fi if are_paths_changed "${packaging_changeset[@]}" ; then PACKAGING_CHANGES="true" -fi \ No newline at end of file +fi diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 850ef14f9bd..e91896eb70c 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -84,7 +84,6 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - # Temporary disabled https://github.com/elastic/beats/issues/37841 - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" command: ".buildkite/scripts/win_unit_tests.ps1" diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index 9cea6564ca8..7c5ee4d1609 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-winlogbeat" + env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_WIN_10: "family/general-windows-10" @@ -16,8 +18,8 @@ steps: - input: "Input Parameters" key: "input-run-all-stages" fields: - - select: "Packetbeat - runWinlogbeat" - key: "runWinlogbeat" + - select: "Winlogbeat - run_winlogbeat" + key: "run_winlogbeat" options: - label: "True" value: "true" @@ -33,3 +35,6 @@ steps: - label: ":linux: Load dynamic winlogbeat pipeline" key: "winlogbeat-pipeline" command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"