devel: plugin replay

build.env

@@ -13,7 +13,7 @@ export IMAGE_EXPIRE_TIME="${EXPIRE:-1w}"
 
 declare -A BUILD_PLATFORMS=()
 BUILD_PLATFORMS+=( ["linux-amd64"]="linux/amd64" )
-BUILD_PLATFORMS+=( ["linux-arm64"]="linux/arm64" )
+#BUILD_PLATFORMS+=( ["linux-arm64"]="linux/arm64" )
 #BUILD_PLATFORMS+=( ["linux-ppc64le"]="linux/ppc64le" )
 #BUILD_PLATFORMS+=( ["linux-s390x"]="linux/s390x" )
 

openshift-tests-provider-cert/plugin/executor.sh

@@ -139,7 +139,7 @@ run_plugin_upgrade() {
 
     else
         os_log_info "[executor] Creating pass JUnit files due the execution mode != upgrade"
-        create_junit_with_msg "pass" "[opct][pass] ignoring upgrade mode on RUN_MODE=[${RUN_MODE-}]." "upgrade"
+        create_junit_with_msg "skipped" "[opct] run plugin upgrade conformance RUN_MODE=[${RUN_MODE-}]." "upgrade"
     fi
 }
 
@@ -152,17 +152,27 @@ run_plugin_upgrade() {
 run_plugins_conformance_replay_config() {
     # Saving failures to replay
     os_log_info "[executor][PluginID#${PLUGIN_ID}] creating failed replay file"
-    junit_file=$(ls ${RESULTS_DIR}/junit_e2e_*.xml)
+    junit_file="$(ls "${RESULTS_DIR}"/junit_e2e_*.xml)"
     if [[ -f "${junit_file}" ]]; then
         os_log_info "[executor][PluginID#${PLUGIN_ID}] parsing junit file ${junit_file}"
         yq -p=xml -o=json "${junit_file}" | jq '.testsuite.testcase[] | select(.failure != null) | .["+@name"]' | sort > "/tmp/failures-${PLUGIN_ID}.txt"
 
-        os_log_info "[executor][PluginID#${PLUGIN_ID}] dump failures ${junit_file}"
+        os_log_info "[executor][PluginID#${PLUGIN_ID}] dump failures from ${junit_file}"
         cat "/tmp/failures-${PLUGIN_ID}.txt"
 
+        # filter for valid tests only (removing 'monitor' and others not included in the suite)
+        os_log_info "[executor][PluginID#${PLUGIN_ID}] filter failures for suite only"
+        while read -r line; do
+            grep ${line} "${RESULTS_DIR}/suite-${CERT_TEST_SUITE/\/}.list" >> "/tmp/failures-${PLUGIN_ID}-suite.txt" || true
+        done < "/tmp/failures-${PLUGIN_ID}.txt"
+
+        os_log_info "[executor][PluginID#${PLUGIN_ID}] dump filtered failures"
+        cat "/tmp/failures-${PLUGIN_ID}-suite.txt"
+
         failed_config="plugin-failures-${PLUGIN_ID}"
         os_log_info "[executor][PluginID#${PLUGIN_ID}] saving failures to config map ${failed_config}"
-        ${UTIL_OC_BIN} create configmap -n "${ENV_POD_NAMESPACE:-sonobuoy}" "plugin-failures-${PLUGIN_ID}" --from-file=replay.list="/tmp/failures-${PLUGIN_ID}.txt"
+        ${UTIL_OC_BIN} create configmap -n "${ENV_POD_NAMESPACE:-sonobuoy}" "plugin-failures-${PLUGIN_ID}" --from-file=replay.list="/tmp/failures-${PLUGIN_ID}-suite.txt" || true
+
     else
         os_log_info "[executor][PluginID#${PLUGIN_ID}]WARNING: unable to find junit file to create failed replay"
     fi
@@ -463,7 +473,7 @@ EOF
         rm -vf "${REPLAY_CONFIG_FILE}"
     fi
     os_log_info "[openshift-tests-plugin][replay][20] Consolidating /tmp/failures-plugin-*"
-    cat /tmp/failures-plugin-* "${REPLAY_CONFIG_FILE}"
+    cat /tmp/failures-plugin-* | sort > "${REPLAY_CONFIG_FILE}"
 
     if [[ -s "${REPLAY_CONFIG_FILE}" ]]; then
         os_log_info "[openshift-tests-plugin][replay] Running e2e from custom config ${REPLAY_CONFIG_FILE}"
@@ -475,6 +485,7 @@ EOF
         replay_save
     else
         os_log_info "[openshift-tests-plugin][replay] replay file not found or is empty, skipping execution: ${REPLAY_CONFIG_FILE}"
+        create_junit_with_msg "skipped" "[opct] run plugin openshift-tests-replay" "replay"
     fi
 }
 

openshift-tests-provider-cert/plugin/global_fn.sh

@@ -79,11 +79,12 @@ init_config() {
         PLUGIN_NAME="${PLUGIN_NAME_TESTS_REPLAY}"
         CERT_TEST_SUITE="${OPENSHIFT_TESTS_SUITE_TESTS_REPLAY}"
         PLUGIN_BLOCKED_BY+=("${PLUGIN_NAME_OPENSHIFT_CONFORMANCE}")
+        export CERT_TEST_COUNT_OVERRIDE=0
 
     elif [[ "${PLUGIN_ID:-}" == "${PLUGIN_ID_OPENSHIFT_ARTIFACTS_COLLECTOR}" ]]
     then
         PLUGIN_NAME="${PLUGIN_NAME_OPENSHIFT_ARTIFACTS_COLLECTOR}"
-        PLUGIN_BLOCKED_BY+=("${PLUGIN_NAME_OPENSHIFT_CONFORMANCE}")
+        PLUGIN_BLOCKED_BY+=("${PLUGIN_NAME_TESTS_REPLAY}")
 
     else
         err="[init_config] Unknown value for PLUGIN_ID=[${PLUGIN_ID:-}]"
@@ -136,6 +137,15 @@ update_config() {
     then
         CERT_TEST_COUNT="$(${UTIL_OTESTS_BIN} run --dry-run "${CERT_TEST_SUITE}" | wc -l)"
     fi
+    # Counter override
+    if [[ -n "${CERT_TEST_COUNT_OVERRIDE-}" ]]; then
+        os_log_info "[update_config] Overriding test count from [${CERT_TEST_COUNT}] to [${CERT_TEST_COUNT_OVERRIDE}]"
+        CERT_TEST_COUNT=${CERT_TEST_COUNT_OVERRIDE}
+    fi
+    if [[ ${DEV_TESTS_COUNT-} -ne 0 ]]; then
+        os_log_info "[update_config] Overriding test count from [${CERT_TEST_COUNT}] to [${DEV_TESTS_COUNT}]"
+        CERT_TEST_COUNT=${DEV_TESTS_COUNT}
+    fi
     os_log_info "[update_config] Total tests found: [${CERT_TEST_COUNT}]"
 }
 
@@ -172,24 +182,29 @@ create_junit_with_msg() {
     local failures_count
     local junit_file
     local junit_file_type
-    local faliures_payload
+    local test_payload
 
     msg_type="$1"; shift
     msg="$1"; shift
     junit_file_type="${1:-e2e}"
     failures_count=0
+    skipped_count=0
 
     if [[ "${msg_type}" == "failed" ]]; then
         failures_count=1
-        faliures_payload="<failure message=\"\">plugin runtime failed</failure><system-out></system-out>"
+        test_payload="<failure message=\"\">OPCT Plugin runtime: unexpected execution failure. Review the plugin logs.</failure><system-out></system-out>"
+    fi
+    if [[ "${msg_type}" == "skipped" ]]; then
+        skipped_count=1
+        test_payload="<skipped message=\"OPCT Plugin Runtime: test skipped. Review the plugin logs for details.\"/>"
     fi
     junit_file="${RESULTS_DIR}/junit_${junit_file_type}_${msg_type}_$(date +%Y%m%d-%H%M%S).xml"
 
     os_log_info "Creating ${msg_type} JUnit result file [${junit_file}]"
     cat << EOF > "${junit_file}"
-<testsuite name="openshift-tests" tests="1" skipped="0" failures="${failures_count}" time="1.0">
+<testsuite name="openshift-tests" tests="1" skipped="${skipped_count}" failures="${failures_count}" time="1.0">
  <property name="TestVersion" value="v4.1.0"></property>
- <testcase name="${msg}" time="0"> ${faliures_payload:-''}
+ <testcase name="${msg}" time="0"> ${test_payload:-''}
 </testcase>
 </testsuite>
 EOF