From d5d33ceec4be53bfdb095b8fc0b2bd60948d85a3 Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Mon, 16 Jun 2025 16:03:55 +0200 Subject: [PATCH 1/4] Add support for sending failed logs to logdetective New function called 'analyze_logs_by_logdetective()' is used as for 'build' part as for 'test' part. Each function is marked by 'LOGDETECTIVE {BUILD|TEST} tag so we can easily find them. Also parsing logs have been enhanced no parse_output is needed. Let's call it directly. It is used on Fedora. We use podman for building and there is no needed to use complicated functions. Signed-off-by: Petr "Stone" Hracek --- build.sh | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- test.sh | 33 ++++++++++++++++++++++++++++++-- 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index edea96a..b7c898a 100755 --- a/build.sh +++ b/build.sh @@ -65,6 +65,31 @@ parse_output () (exit $rc) } +analyze_logs_by_logdetective() { + local log_file_name="$1" + echo "Sending failed log by fpaste command to paste bin." + paste_bin_link=$(fpaste "$log_file_name") + # shellcheck disable=SC2181 + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to send log file to private bin: ${log_file_name}" + return + fi + # pastebin link is "https://paste.centos.org/view/ee98ba05" + # We need a raw link that is "https://paste.centos.org/view/raw/ee98ba05" + raw_paste_bin_link="${paste_bin_link//view/view\/raw}" + echo "Sending log file to logdetective server: ${raw_paste_bin_link}" + echo "-------- LOGDETECTIVE BUILD LOG ANALYSIS START --------" + # shellcheck disable=SC2181 + if ! curl -k --insecure --header "Content-Type: application/json" --request POST --data "{\"url\":\"${raw_paste_bin_link}\"}" "$LOGDETECTIVE_SERVER/analyze" > /tmp/logdetective_output.txt; then + echo "ERROR: Failed to analyze log file by logdetective server." + cat "/tmp/logdetective_output.txt" + echo "-------- LOGDETECTIVE BUILD LOG ANALYSIS FAILED --------" + return + fi + jq -rC '.explanation.text' < "/tmp/logdetective_output.txt" + echo "-------- LOGDETECTIVE BUILD LOG ANALYSIS FINISHED --------" +} + # "best-effort" cleanup of image function clean_image { for id_file in .image-id .image-id-from; do @@ -191,10 +216,37 @@ function docker_build_with_version { if [[ "$SKIP_SQUASH" -eq 0 ]] && [[ "$is_podman" -eq 1 ]]; then BUILD_OPTIONS+=" --squash" fi + + command="docker build ${BUILD_OPTIONS} -f $dockerfile ${DOCKER_BUILD_CONTEXT}" + echo "-> building using $command" + + tmp_file=$(mktemp "/tmp/${dir}-${OS}.XXXXXX") + $command 2>&1 | tee "$tmp_file" + cat "$tmp_file" + last_row=$(< "$tmp_file" tail -n 1) + commit_message=$(< "$tmp_file" tail -n 3) + # Structure of log build is as follows: + # COMMIT + # --> e191d12b5928 + # e191d12b5928360dd6024fe80d31e08f994d42577f76b9b143e014749afc8ab4 + # shellcheck disable=SC2016 + if [[ "$commit_message" == *"COMMIT"* ]] && [[ "$last_row" =~ (^-->)?(Using cache )?[a-fA-F0-9]+$ ]]; then + IMAGE_ID="$last_row" + else + # Do not fail in case of sending log to pastebin or logdetective fails. + echo "Analyse logs by logdetective, why it failed." + analyze_logs_by_logdetective "${tmp_file}" + exit 1 + fi + + rm -f "$tmp_file" + # shellcheck disable=SC2016 - parse_output 'docker build '"$BUILD_OPTIONS"' -f "$dockerfile" "${DOCKER_BUILD_CONTEXT}"' \ - "tail -n 1 | awk '/Successfully built|(^--> )?(Using cache )?[a-fA-F0-9]+$/{print \$NF}'" \ - IMAGE_ID + +# parse_output 'docker build '"$BUILD_OPTIONS"' -f "$dockerfile" "${DOCKER_BUILD_CONTEXT}"' \ +# "tail -n 1 | awk '/Successfully built|(^--> )?(Using cache )?[a-fA-F0-9]+$/{print \$NF}'" \ +# IMAGE_ID +# analyze_logs_by_logdetective "$?" "${tmp_file}" echo "$IMAGE_ID" > .image-id } diff --git a/test.sh b/test.sh index 770de04..3b329a1 100755 --- a/test.sh +++ b/test.sh @@ -24,6 +24,31 @@ failed_version() { return "$result" } +analyze_logs_by_logdetective() { + local log_file_name="$1" + echo "Sending failed log by fpaste command to paste bin." + paste_bin_link=$(fpaste "$log_file_name") + # shellcheck disable=SC2181 + if [[ $? -ne 0 ]]; then + echo "ERROR: Failed to send log file to private bin: ${log_file_name}" + return + fi + # pastebin link is "https://paste.centos.org/view/ee98ba05" + # We need a raw link that is "https://paste.centos.org/view/raw/ee98ba05" + raw_paste_bin_link="${paste_bin_link//view/view\/raw}" + echo "Sending log file to logdetective server: ${raw_paste_bin_link}" + echo "-------- LOGDETECTIVE TEST LOG ANALYSIS START --------" + # shellcheck disable=SC2181 + if ! curl -k --insecure --header "Content-Type: application/json" --request POST --data "{\"url\":\"${raw_paste_bin_link}\"}" "$LOGDETECTIVE_SERVER/analyze" > /tmp/logdetective_test_output.txt; then + echo "ERROR: Failed to analyze log file by logdetective server." + cat "/tmp/logdetective_test_output.txt" + echo "-------- LOGDETECTIVE TEST LOG ANALYSIS FAILED --------" + return + fi + jq -rC '.explanation.text' < "/tmp/logdetective_test_output.txt" + echo "-------- LOGDETECTIVE TEST LOG ANALYSIS FINISHED --------" +} + # This adds backwards compatibility if only single version needs to be testing # In CI we would like to test single version but VERSIONS= means, that nothing is tested # make test TARGET= VERSIONS= ... checks single version for CLI @@ -47,8 +72,12 @@ for dir in ${VERSIONS}; do fi if [ -n "${TEST_MODE}" ]; then - VERSION=$dir test/run - failed_version "$?" "$dir" + tmp_file=$(mktemp "/tmp/${IMAGE_NAME}-${OS}-${dir}.XXXXXX") + VERSION=$dir test/run 2>&1 | tee "$tmp_file" + ret_code=$? + analyze_logs_by_logdetective "$tmp_file" + failed_version "$ret_code" "$dir" + rm -f "$tmp_file" fi if [ -n "${TEST_OPENSHIFT_4}" ]; then From 428c450523f2ad145bddc7b5a79accc970205b6f Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Tue, 24 Jun 2025 10:53:01 +0200 Subject: [PATCH 2/4] Enable logdetective only for RHEL systems Signed-off-by: Petr "Stone" Hracek --- build.sh | 12 ++++++------ test.sh | 6 +++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/build.sh b/build.sh index b7c898a..e30a6f4 100755 --- a/build.sh +++ b/build.sh @@ -219,23 +219,23 @@ function docker_build_with_version { command="docker build ${BUILD_OPTIONS} -f $dockerfile ${DOCKER_BUILD_CONTEXT}" echo "-> building using $command" - tmp_file=$(mktemp "/tmp/${dir}-${OS}.XXXXXX") $command 2>&1 | tee "$tmp_file" cat "$tmp_file" last_row=$(< "$tmp_file" tail -n 1) - commit_message=$(< "$tmp_file" tail -n 3) # Structure of log build is as follows: # COMMIT # --> e191d12b5928 # e191d12b5928360dd6024fe80d31e08f994d42577f76b9b143e014749afc8ab4 # shellcheck disable=SC2016 - if [[ "$commit_message" == *"COMMIT"* ]] && [[ "$last_row" =~ (^-->)?(Using cache )?[a-fA-F0-9]+$ ]]; then + if [[ "$last_row" =~ (^-->)?(Using cache )?[a-fA-F0-9]+$ ]]; then IMAGE_ID="$last_row" else - # Do not fail in case of sending log to pastebin or logdetective fails. - echo "Analyse logs by logdetective, why it failed." - analyze_logs_by_logdetective "${tmp_file}" + if [[ "${OS}" == "rhel8" ]] || [[ "${OS}" == "rhel9" ]] || [[ "${OS}" == "rhel10" ]]; then + # Do not fail in case of sending log to pastebin or logdetective fails. + echo "Analyse logs by logdetective, why it failed." + analyze_logs_by_logdetective "${tmp_file}" + fi exit 1 fi diff --git a/test.sh b/test.sh index 3b329a1..4f81087 100755 --- a/test.sh +++ b/test.sh @@ -25,6 +25,7 @@ failed_version() { } analyze_logs_by_logdetective() { + set +e local log_file_name="$1" echo "Sending failed log by fpaste command to paste bin." paste_bin_link=$(fpaste "$log_file_name") @@ -43,6 +44,7 @@ analyze_logs_by_logdetective() { echo "ERROR: Failed to analyze log file by logdetective server." cat "/tmp/logdetective_test_output.txt" echo "-------- LOGDETECTIVE TEST LOG ANALYSIS FAILED --------" + set -e return fi jq -rC '.explanation.text' < "/tmp/logdetective_test_output.txt" @@ -75,7 +77,9 @@ for dir in ${VERSIONS}; do tmp_file=$(mktemp "/tmp/${IMAGE_NAME}-${OS}-${dir}.XXXXXX") VERSION=$dir test/run 2>&1 | tee "$tmp_file" ret_code=$? - analyze_logs_by_logdetective "$tmp_file" + if [[ "${OS}" == "rhel8" ]] || [[ "${OS}" == "rhel9" ]] || [[ "${OS}" == "rhel10" ]]; then + analyze_logs_by_logdetective "$tmp_file" + fi failed_version "$ret_code" "$dir" rm -f "$tmp_file" fi From 1986965dda5a6e0a9868c271f8e142ec2a2aa8ce Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Tue, 24 Jun 2025 14:36:42 +0200 Subject: [PATCH 3/4] Fix pipeline failure Signed-off-by: Petr "Stone" Hracek --- test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test.sh b/test.sh index 4f81087..a95ed07 100755 --- a/test.sh +++ b/test.sh @@ -48,6 +48,7 @@ analyze_logs_by_logdetective() { return fi jq -rC '.explanation.text' < "/tmp/logdetective_test_output.txt" + set -e echo "-------- LOGDETECTIVE TEST LOG ANALYSIS FINISHED --------" } @@ -74,9 +75,12 @@ for dir in ${VERSIONS}; do fi if [ -n "${TEST_MODE}" ]; then - tmp_file=$(mktemp "/tmp/${IMAGE_NAME}-${OS}-${dir}.XXXXXX") + set -o pipefail + tmp_file=$(mktemp "/tmp/${OS}-${dir}.XXXXXX") VERSION=$dir test/run 2>&1 | tee "$tmp_file" ret_code=$? + set +o pipefail + cat "${tmp_file}" if [[ "${OS}" == "rhel8" ]] || [[ "${OS}" == "rhel9" ]] || [[ "${OS}" == "rhel10" ]]; then analyze_logs_by_logdetective "$tmp_file" fi From 195784a161628c1438ecd8929628f6a55c8a2cc7 Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Thu, 3 Jul 2025 15:45:09 +0200 Subject: [PATCH 4/4] Store logdetective output to tempfile and at the end store it to $TMT_TEST_DATA directory so we see it in Testing Farm results Signed-off-by: Petr "Stone" Hracek --- build.sh | 20 +++++++++++++++++--- test.sh | 17 ++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/build.sh b/build.sh index e30a6f4..d86b6e5 100755 --- a/build.sh +++ b/build.sh @@ -66,6 +66,8 @@ parse_output () } analyze_logs_by_logdetective() { + # logdetective should not break the build functionality + set +e local log_file_name="$1" echo "Sending failed log by fpaste command to paste bin." paste_bin_link=$(fpaste "$log_file_name") @@ -79,14 +81,26 @@ analyze_logs_by_logdetective() { raw_paste_bin_link="${paste_bin_link//view/view\/raw}" echo "Sending log file to logdetective server: ${raw_paste_bin_link}" echo "-------- LOGDETECTIVE BUILD LOG ANALYSIS START --------" + logdetective_build_file=$(mktemp "/tmp/logdetective_build.XXXXXX") # shellcheck disable=SC2181 - if ! curl -k --insecure --header "Content-Type: application/json" --request POST --data "{\"url\":\"${raw_paste_bin_link}\"}" "$LOGDETECTIVE_SERVER/analyze" > /tmp/logdetective_output.txt; then + if ! curl -k --insecure --header "Content-Type: application/json" --request POST --data "{\"url\":\"${raw_paste_bin_link}\"}" "$LOGDETECTIVE_SERVER/analyze" >> "${logdetective_build_file}"; then echo "ERROR: Failed to analyze log file by logdetective server." - cat "/tmp/logdetective_output.txt" + cat "${logdetective_build_file}" echo "-------- LOGDETECTIVE BUILD LOG ANALYSIS FAILED --------" + set -e return fi - jq -rC '.explanation.text' < "/tmp/logdetective_output.txt" + set -e + jq -rC '.explanation.text' < "${logdetective_build_file}" + # This part of code is from https://github.com/teemtee/tmt/blob/main/tmt/steps/scripts/tmt-file-submit + if [ -z "$TMT_TEST_PIDFILE" ]; then + echo "File submit to data dir can be used only in the context of a running test." + return + fi + # This variable is set by tmt + [ -d "$TMT_TEST_DATA" ] || mkdir -p "$TMT_TEST_DATA" + cp -f "${logdetective_build_file}" "$TMT_TEST_DATA" + echo "File '${logdetective_build_file}' stored to '$TMT_TEST_DATA'." echo "-------- LOGDETECTIVE BUILD LOG ANALYSIS FINISHED --------" } diff --git a/test.sh b/test.sh index a95ed07..585fe16 100755 --- a/test.sh +++ b/test.sh @@ -25,6 +25,7 @@ failed_version() { } analyze_logs_by_logdetective() { + # logdetective should not break the test functionality set +e local log_file_name="$1" echo "Sending failed log by fpaste command to paste bin." @@ -39,16 +40,26 @@ analyze_logs_by_logdetective() { raw_paste_bin_link="${paste_bin_link//view/view\/raw}" echo "Sending log file to logdetective server: ${raw_paste_bin_link}" echo "-------- LOGDETECTIVE TEST LOG ANALYSIS START --------" + logdetective_test_file=$(mktemp "/tmp/logdetective_test.XXXXXX") # shellcheck disable=SC2181 - if ! curl -k --insecure --header "Content-Type: application/json" --request POST --data "{\"url\":\"${raw_paste_bin_link}\"}" "$LOGDETECTIVE_SERVER/analyze" > /tmp/logdetective_test_output.txt; then + if ! curl -k --insecure --header "Content-Type: application/json" --request POST --data "{\"url\":\"${raw_paste_bin_link}\"}" "$LOGDETECTIVE_SERVER/analyze" >> "${logdetective_test_file}"; then echo "ERROR: Failed to analyze log file by logdetective server." - cat "/tmp/logdetective_test_output.txt" + cat "${logdetective_test_file}" echo "-------- LOGDETECTIVE TEST LOG ANALYSIS FAILED --------" set -e return fi - jq -rC '.explanation.text' < "/tmp/logdetective_test_output.txt" set -e + jq -rC '.explanation.text' < "${logdetective_test_file}" + # This part of code is from https://github.com/teemtee/tmt/blob/main/tmt/steps/scripts/tmt-file-submit + if [ -z "$TMT_TEST_PIDFILE" ]; then + echo "File submit to data dir can be used only in the context of a running test." + return + fi + # This variable is set by tmt + [ -d "$TMT_TEST_DATA" ] || mkdir -p "$TMT_TEST_DATA" + cp -f "${logdetective_test_file}" "$TMT_TEST_DATA" + echo "File '${logdetective_test_file}' stored to '$TMT_TEST_DATA'." echo "-------- LOGDETECTIVE TEST LOG ANALYSIS FINISHED --------" }